src/main/scala/s3/website/Diff.scala in s3_website-2.0.1 vs src/main/scala/s3/website/Diff.scala in s3_website-2.1.0
- old
+ new
@@ -17,11 +17,11 @@
persistenceError: Future[Option[ErrorReport]]
)
object Diff {
- type UploadBatch = Future[Either[ErrorReport, Seq[LocalFile]]]
+ type UploadBatch = Future[Either[ErrorReport, Seq[Upload]]]
def resolveDiff(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
(implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] =
if (LocalFileDatabase.hasRecords) resolveDiffAgainstLocalDb(s3FilesFuture)
else resolveDiffAgainstGetBucketResponse(s3FilesFuture)
@@ -33,43 +33,43 @@
Try {
val s3KeyIndex = s3Files.map(_.s3Key).toSet
val s3Md5Index = s3Files.map(_.md5).toSet
val siteFiles = Files.listSiteFiles
val existsOnS3 = (f: File) => s3KeyIndex contains site.resolveS3Key(f)
- val isChangedOnS3 = (localFile: LocalFile) => !(s3Md5Index contains localFile.md5.get)
- val newFiles = siteFiles collect {
- case file if !existsOnS3(file) => LocalFile(file, NewFile)
+ val isChangedOnS3 = (upload: Upload) => !(s3Md5Index contains upload.md5.get)
+ val newUploads = siteFiles collect {
+ case file if !existsOnS3(file) => Upload(file, NewFile, reasonForUpload = "the file is missing from S3")
}
- val changedFiles = siteFiles collect {
- case file if existsOnS3(file) => LocalFile(file, FileUpdate)
+ val changedUploads = siteFiles collect {
+ case file if existsOnS3(file) => Upload(file, FileUpdate, reasonForUpload = "the S3 bucket has different contents for this file")
} filter isChangedOnS3
val unchangedFiles = {
- val newOrChangedFiles = (changedFiles ++ newFiles).map(_.originalFile).toSet
+ val newOrChangedFiles = (changedUploads ++ newUploads).map(_.originalFile).toSet
siteFiles.filterNot(f => newOrChangedFiles contains f)
}
- val allFiles: Seq[Either[DbRecord, LocalFile]] = unchangedFiles.map {
+ val recordsAndUploads: Seq[Either[DbRecord, Upload]] = unchangedFiles.map {
f => Left(DbRecord(f))
- } ++ (changedFiles ++ newFiles).map {
+ } ++ (changedUploads ++ newUploads).map {
Right(_)
}
- LocalFileDatabase persist allFiles
- allFiles
+ LocalFileDatabase persist recordsAndUploads
+ recordsAndUploads
} match {
case Success(ok) => Right(ok)
case Failure(err) => Left(ErrorReport(err))
}
}
}
- def collectResult[B](pf: PartialFunction[Either[DbRecord, LocalFile],B]) =
+ def collectResult[B](pf: PartialFunction[Either[DbRecord, Upload],B]) =
diffAgainstS3.map { errorOrDiffSource =>
errorOrDiffSource.right map (_ collect pf)
}
val unchanged = collectResult {
case Left(dbRecord) => dbRecord.s3Key
}
val uploads: UploadBatch = collectResult {
- case Right(localFile) => localFile
+ case Right(upload) => upload
}
Right(Diff(unchanged, uploads :: Nil, persistenceError = Future(None)))
}
def resolveDeletes(diff: Diff, s3Files: Future[Either[ErrorReport, Seq[S3File]]], redirects: Seq[Redirect])
@@ -110,30 +110,32 @@
databaseIndices <- loadDbFromFile(dbFile)
} yield databaseIndices.fullIndex.headOption.isDefined) getOrElse false
def resolveDiffAgainstLocalDb(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
(implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] = {
- val localDiff: Either[ErrorReport, Seq[Either[DbRecord, LocalFile]]] =
+ val localDiff: Either[ErrorReport, Seq[Either[DbRecord, Upload]]] =
(for {
dbFile <- getOrCreateDbFile
databaseIndices <- loadDbFromFile(dbFile)
} yield {
val siteFiles = Files.listSiteFiles
- val recordsOrChangedFiles = siteFiles.foldLeft(Seq(): Seq[Either[DbRecord, LocalFile]]) { (localFiles, file) =>
+ val recordsOrUploads = siteFiles.foldLeft(Seq(): Seq[Either[DbRecord, Upload]]) { (recordsOrUps, file) =>
val truncatedKey = TruncatedDbRecord(file)
val fileIsUnchanged = databaseIndices.truncatedIndex contains truncatedKey
if (fileIsUnchanged)
- localFiles :+ Left(databaseIndices.fullIndex find (_.truncated == truncatedKey) get)
+ recordsOrUps :+ Left(databaseIndices.fullIndex find (_.truncated == truncatedKey) get)
else {
+ val isUpdate = databaseIndices.s3KeyIndex contains truncatedKey.s3Key
+
val uploadType =
- if (databaseIndices.s3KeyIndex contains truncatedKey.s3Key) FileUpdate
+ if (isUpdate) FileUpdate
else NewFile
- localFiles :+ Right(LocalFile(file, uploadType))
+ recordsOrUps :+ Right(Upload(file, uploadType, reasonForUpload(truncatedKey, databaseIndices, isUpdate)))
}
}
- logger.debug(s"Discovered ${siteFiles.length} files on the local site, of which ${recordsOrChangedFiles count (_.isRight)} are new or changed")
- recordsOrChangedFiles
+ logger.debug(s"Discovered ${siteFiles.length} files on the local site, of which ${recordsOrUploads count (_.isRight)} are new or changed")
+ recordsOrUploads
}) match {
case Success(ok) => Right(ok)
case Failure(err) => Left(ErrorReport(err))
}
@@ -144,23 +146,23 @@
val uploadsAccordingToLocalDiff = localDiffResult collect {
case Right(f) => f
}
- val changesMissedByLocalDiff: Future[Either[ErrorReport, Seq[LocalFile]]] = s3FilesFuture.map { errorOrS3Files =>
+ val changesMissedByLocalDiff: Future[Either[ErrorReport, Seq[Upload]]] = s3FilesFuture.map { errorOrS3Files =>
for (s3Files <- errorOrS3Files.right) yield {
val remoteS3Keys = s3Files.map(_.s3Key).toSet
val localS3Keys = unchangedAccordingToLocalDiff.map(_.s3Key).toSet
val localMd5 = unchangedAccordingToLocalDiff.map(_.uploadFileMd5).toSet
def isChangedOnS3(s3File: S3File) = (localS3Keys contains s3File.s3Key) && !(localMd5 contains s3File.md5)
val changedOnS3 = s3Files collect {
case s3File if isChangedOnS3(s3File) =>
- LocalFile(site resolveFile s3File, FileUpdate)
+ Upload(site resolveFile s3File, FileUpdate, reasonForUpload = "someone else has modified the file on the S3 bucket")
}
val missingFromS3 = localS3Keys collect {
case localS3Key if !(remoteS3Keys contains localS3Key) =>
- LocalFile(site resolveFile localS3Key, NewFile)
+ Upload(site resolveFile localS3Key, NewFile, reasonForUpload = "someone else has removed the file from the S3 bucket")
}
changedOnS3 ++ missingFromS3
}
}
@@ -175,52 +177,70 @@
}
val unchangedFilesFinal = errorOrDiffAgainstS3 map {
_ fold (
(error: ErrorReport) => Left(error),
- (syncResult: (Seq[DbRecord], Seq[LocalFile])) => Right(syncResult._1)
+ (syncResult: (Seq[DbRecord], Seq[Upload])) => Right(syncResult._1)
)
}
- val changedAccordingToS3Diff = errorOrDiffAgainstS3.map {
+ val uploadsAccordingToS3Diff = errorOrDiffAgainstS3.map {
_ fold (
(error: ErrorReport) => Left(error),
- (syncResult: (Seq[DbRecord], Seq[LocalFile])) => Right(syncResult._2)
+ (syncResult: (Seq[DbRecord], Seq[Upload])) => Right(syncResult._2)
)
}
val persistenceError: Future[Either[ErrorReport, _]] = for {
unchanged <- unchangedFilesFinal
- changedAccordingToS3 <- changedAccordingToS3Diff
+ uploads <- uploadsAccordingToS3Diff
} yield
for {
records1 <- unchanged.right
- records2 <- changedAccordingToS3.right
+ records2 <- uploads.right
} yield
persist(records1.map(Left(_)) ++ records2.map(Right(_)) ++ uploadsAccordingToLocalDiff.map(Right(_))) match {
case Success(_) => Unit
case Failure(err) => ErrorReport(err)
}
Diff(
unchangedFilesFinal map (_.right.map(_ map (_.s3Key))),
- uploads = Future(Right(uploadsAccordingToLocalDiff)) :: changedAccordingToS3Diff :: Nil,
+ uploads = Future(Right(uploadsAccordingToLocalDiff)) :: uploadsAccordingToS3Diff :: Nil,
persistenceError = persistenceError map (_.left.toOption)
)
}
}
+ private def reasonForUpload(truncatedKey: TruncatedDbRecord, databaseIndices: DbIndices, isUpdate: Boolean) = {
+ if (isUpdate) {
+ val lengthChanged = !(databaseIndices.fileLenghtIndex contains truncatedKey.fileLength)
+ val mtimeChanged = !(databaseIndices.lastModifiedIndex contains truncatedKey.fileModified)
+ if (lengthChanged)
+ "file length has changed according to the local database"
+ else if (mtimeChanged)
+ "file mtime has changed according to the local database"
+ else if (mtimeChanged && lengthChanged)
+ "file mtime and length have changed according to the local database"
+ else
+ "programmer error: faulty logic in inferring the reason for upload"
+ }
+ else "file is new according to the local database"
+ }
+
private def getOrCreateDbFile(implicit site: Site, logger: Logger) =
Try {
val dbFile = new File(getTempDirectory, "s3_website_local_db_" + sha256Hex(site.rootDirectory))
if (!dbFile.exists()) logger.debug("Creating a new database in " + dbFile.getName)
dbFile.createNewFile()
dbFile
}
case class DbIndices(
- s3KeyIndex: Set[S3Key],
- truncatedIndex: Set[TruncatedDbRecord],
- fullIndex: Set[DbRecord]
+ s3KeyIndex: Set[S3Key],
+ fileLenghtIndex: Set[Long],
+ lastModifiedIndex: Set[Long],
+ truncatedIndex: Set[TruncatedDbRecord],
+ fullIndex: Set[DbRecord]
)
private def loadDbFromFile(databaseFile: File)(implicit site: Site, logger: Logger): Try[DbIndices] =
Try {
// record format: "s3Key(file.path)|length(file)|mtime(file)|md5Hex(file.encoded)"
@@ -235,27 +255,29 @@
}
.toSet
DbIndices(
s3KeyIndex = fullIndex map (_.s3Key),
truncatedIndex = fullIndex map (TruncatedDbRecord(_)),
- fullIndex
+ fileLenghtIndex = fullIndex map (_.fileLength),
+ lastModifiedIndex = fullIndex map (_.fileModified),
+ fullIndex = fullIndex
)
}
- def persist(recordsOrChangedFiles: Seq[Either[DbRecord, LocalFile]])(implicit site: Site, logger: Logger): Try[Seq[Either[DbRecord, LocalFile]]] =
+ def persist(recordsOrUploads: Seq[Either[DbRecord, Upload]])(implicit site: Site, logger: Logger): Try[Seq[Either[DbRecord, Upload]]] =
getOrCreateDbFile flatMap { dbFile =>
Try {
- val dbFileContents = recordsOrChangedFiles.map { recordOrChangedFile =>
- val record: DbRecord = recordOrChangedFile fold(
+ val dbFileContents = recordsOrUploads.map { recordOrUpload =>
+ val record: DbRecord = recordOrUpload fold(
record => record,
- changedFile => DbRecord(changedFile.s3Key, changedFile.originalFile.length, changedFile.originalFile.lastModified, changedFile.md5.get)
+ upload => DbRecord(upload.s3Key, upload.originalFile.length, upload.originalFile.lastModified, upload.md5.get)
)
record.s3Key :: record.fileLength :: record.fileModified :: record.uploadFileMd5 :: Nil mkString "|"
} mkString "\n"
write(dbFile, dbFileContents)
- recordsOrChangedFiles
+ recordsOrUploads
}
}
}
case class TruncatedDbRecord(s3Key: String, fileLength: Long, fileModified: Long)
@@ -273,8 +295,8 @@
lazy val truncated = TruncatedDbRecord(s3Key, fileLength, fileModified)
}
object DbRecord {
def apply(original: File)(implicit site: Site): DbRecord =
- DbRecord(site resolveS3Key original, original.length, original.lastModified, LocalFile.md5(original).get)
+ DbRecord(site resolveS3Key original, original.length, original.lastModified, Upload.md5(original).get)
}
}
\ No newline at end of file