-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add windowed filename policy (closes #5)
- Loading branch information
Showing
5 changed files
with
169 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
58 changes: 58 additions & 0 deletions
58
src/main/scala/com/snowplowanalytics/WindowedFilenamePolicy.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| package com.snowplowanalytics | ||
|
|
||
| import org.apache.beam.sdk.io.DefaultFilenamePolicy | ||
| import org.apache.beam.sdk.io.FileBasedSink.{FilenamePolicy, OutputFileHints} | ||
| import org.apache.beam.sdk.io.FileSystems | ||
| import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions | ||
| import org.apache.beam.sdk.io.fs.ResourceId | ||
| import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider | ||
| import org.apache.beam.sdk.transforms.windowing.{BoundedWindow, IntervalWindow, PaneInfo} | ||
| import org.joda.time.DateTime | ||
| import org.joda.time.format.DateTimeFormat | ||
|
|
||
| final case class WindowedFilenamePolicy( | ||
| outputDirectory: String, | ||
| outputFilenamePrefix: String, | ||
| shardTemplate: String, | ||
| outputFilenameSuffix: String | ||
| ) extends FilenamePolicy { | ||
| override def windowedFilename( | ||
| shardNumber: Int, | ||
| numShards: Int, | ||
| window: BoundedWindow, | ||
| paneInfo: PaneInfo, | ||
| outputFileHints: OutputFileHints | ||
| ): ResourceId = { | ||
| val outputFile = resolveWithDateTemplates(outputDirectory, window) | ||
| .resolve(outputFilenamePrefix, StandardResolveOptions.RESOLVE_FILE) | ||
| val policy = DefaultFilenamePolicy.fromStandardParameters( | ||
| StaticValueProvider.of(outputFile), shardTemplate, outputFilenameSuffix, windowedWrites = true) | ||
| policy.windowedFilename(shardNumber, numShards, window, paneInfo, outputFileHints) | ||
| } | ||
|
|
||
| override def unwindowedFilename( | ||
| shardNumber: Int, | ||
| numShards: Int, | ||
| outputFileHints: OutputFileHints | ||
| ): ResourceId = throw new UnsupportedOperationException("This policy only supports windowed files") | ||
|
|
||
| private def resolveWithDateTemplates( | ||
| outputDirectory: String, | ||
| window: BoundedWindow | ||
| ): ResourceId = { | ||
| val outputDir = FileSystems.matchNewResource(outputDirectory, isDirectory = true) | ||
|
|
||
| window match { | ||
| case w: IntervalWindow => | ||
| val windowEndTime = w.end.toDateTime | ||
| val outputPath = dateFormat(windowEndTime)(outputDir.toString) | ||
| FileSystems.matchNewResource(outputPath, isDirectory = true) | ||
| case _ => outputDir | ||
| } | ||
| } | ||
|
|
||
| private def dateFormat(t: DateTime): String => String = | ||
| ((s: String) => s.replace("YYYY", DateTimeFormat.forPattern("YYYY").print(t))) andThen | ||
| ((s: String) => s.replace("MM", DateTimeFormat.forPattern("MM").print(t))) andThen | ||
| ((s: String) => s.replace("DD", DateTimeFormat.forPattern("DD").print(t))) | ||
| } |
68 changes: 68 additions & 0 deletions
68
src/test/scala/com/snowplowanalytics/WindowedFilenamePolicySpec.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| package com.snowplowanalytics | ||
|
|
||
| import java.io.File | ||
| import java.nio.file.Files | ||
|
|
||
| import org.apache.beam.sdk.io.FileBasedSink.OutputFileHints | ||
| import org.apache.beam.sdk.io.LocalResources | ||
| import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions | ||
| import org.apache.beam.sdk.transforms.windowing.{BoundedWindow, IntervalWindow, PaneInfo} | ||
| import org.joda.time.DateTime | ||
|
|
||
| import org.mockito.Mockito.when | ||
| import org.scalatest.FreeSpec | ||
| import org.scalatest.Matchers._ | ||
| import org.scalatest.mockito.MockitoSugar | ||
|
|
||
| class WindowedFilenamePolicySpec extends FreeSpec with MockitoSugar { | ||
| object TestOutputFileHints extends OutputFileHints { | ||
| override def getMimeType: String = "" | ||
| override def getSuggestedFilenameSuffix: String = "" | ||
| } | ||
|
|
||
| "the WindowedFilenamePolicy" - { | ||
| "make a windowedFilename function available" - { | ||
| "which produces a windowed filename" in { | ||
| val outputDirectory = LocalResources.fromFile(tempDir(), isDirectory = true) | ||
| .resolve("WindowedFilenamePolicy", StandardResolveOptions.RESOLVE_DIRECTORY) | ||
| val window = mock[BoundedWindow] | ||
| val paneInfo = PaneInfo.createPane(false, true, PaneInfo.Timing.ON_TIME, 0, 0) | ||
| val policy = WindowedFilenamePolicy(outputDirectory.toString, "out", "-SSS-NNN", ".txt") | ||
|
|
||
| val filename = policy.windowedFilename(1, 1, window, paneInfo, TestOutputFileHints) | ||
|
|
||
| filename.getFilename shouldEqual "out-001-001.txt" | ||
| } | ||
| "which produces a dynamic filename" in { | ||
| val outputDirectory = LocalResources.fromFile(tempDir(), isDirectory = true) | ||
| .resolve("YYYY/MM/dd/HH", StandardResolveOptions.RESOLVE_DIRECTORY) | ||
| val paneInfo = PaneInfo.createPane(false, true, PaneInfo.Timing.ON_TIME, 0, 0) | ||
| val window = mock[IntervalWindow] | ||
| val windowBegin = new DateTime(2018, 1, 8, 10, 55, 0).toInstant | ||
| val windowEnd = new DateTime(2018, 1, 8, 10, 56, 0).toInstant | ||
| when(window.maxTimestamp).thenReturn(windowEnd) | ||
| when(window.start).thenReturn(windowBegin) | ||
| when(window.end).thenReturn(windowEnd) | ||
| val policy = WindowedFilenamePolicy(outputDirectory.toString, "out", "-SSS-NNN", ".txt") | ||
|
|
||
| val filename = policy.windowedFilename(1, 1, window, paneInfo, TestOutputFileHints) | ||
|
|
||
| filename.getCurrentDirectory.toString should endWith("2018/01/08/10/") | ||
| filename.getFilename shouldEqual "out-001-001.txt" | ||
| } | ||
| } | ||
| "make a unwindowedFilename function available" - { | ||
| "which throws an unsupported operation" in { | ||
| val outputDirectory = LocalResources.fromFile(tempDir(), isDirectory = true) | ||
| .resolve("WindowedFilenamePolicy", StandardResolveOptions.RESOLVE_DIRECTORY) | ||
| val policy = WindowedFilenamePolicy(outputDirectory.toString, "out", "-SSS-NNN", ".txt") | ||
|
|
||
| an [UnsupportedOperationException] should be thrownBy | ||
| policy.unwindowedFilename(1, 1, TestOutputFileHints) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private def tempDir(): File = | ||
| Files.createTempDirectory("WindowedFilenamePolicy").toFile | ||
| } |