src/main/java/org/embulk/input/gcs/GcsFileInputPlugin.java in embulk-input-gcs-0.2.2 vs src/main/java/org/embulk/input/gcs/GcsFileInputPlugin.java in embulk-input-gcs-0.2.3
- old
+ new
@@ -57,10 +57,14 @@
@Config("last_path")
@ConfigDefault("null")
Optional<String> getLastPath();
+ @Config("incremental")
+ @ConfigDefault("true")
+ boolean getIncremental();
+
@Config("auth_method")
@ConfigDefault("\"private_key\"")
AuthMethod getAuthMethod();
@Config("service_account_email")
@@ -177,18 +181,20 @@
control.run(taskSource, taskCount);
ConfigDiff configDiff = Exec.newConfigDiff();
List<String> files = new ArrayList<String>(task.getFiles());
- if (files.isEmpty()) {
- // keep the last value if any
- if (task.getLastPath().isPresent()) {
- configDiff.set("last_path", task.getLastPath().get());
+ if (task.getIncremental()) {
+ if (files.isEmpty()) {
+ // keep the last value if any
+ if (task.getLastPath().isPresent()) {
+ configDiff.set("last_path", task.getLastPath().get());
+ }
}
- }
- else {
- Collections.sort(files);
- configDiff.set("last_path", files.get(files.size() - 1));
+ else {
+ Collections.sort(files);
+ configDiff.set("last_path", files.get(files.size() - 1));
+ }
}
return configDiff;
}