src/main/java/org/embulk/input/gcs/GcsFileInputPlugin.java in embulk-input-gcs-0.1.13 vs src/main/java/org/embulk/input/gcs/GcsFileInputPlugin.java in embulk-input-gcs-0.2.0

- old
+ new

@@ -50,11 +50,12 @@ { @Config("bucket") String getBucket(); @Config("path_prefix") - String getPathPrefix(); + @ConfigDefault("null") + Optional<String> getPathPrefix(); @Config("last_path") @ConfigDefault("null") Optional<String> getLastPath(); @@ -82,10 +83,12 @@ @Config("json_keyfile") @ConfigDefault("null") Optional<LocalFile> getJsonKeyfile(); + @Config("paths") + @ConfigDefault("[]") List<String> getFiles(); void setFiles(List<String> files); @ConfigInject BufferAllocator getBufferAllocator(); @@ -129,12 +132,17 @@ } } Storage client = newGcsClient(task, newGcsAuth(task)); - // list files recursively - task.setFiles(listFiles(task, client)); + // list files recursively if path_prefix is specified + if (task.getPathPrefix().isPresent()) { + task.setFiles(listFiles(task, client)); + } + if (task.getFiles().isEmpty()) { + throw new ConfigException("No file is found. Fix path_prefix or specify paths directly"); + } // number of processors is same with number of files return resume(task.dump(), task.getFiles().size(), control); } private GcsAuthentication newGcsAuth(PluginTask task) @@ -212,10 +220,10 @@ public List<String> listFiles(PluginTask task, Storage client) { String bucket = task.getBucket(); - return listGcsFilesByPrefix(client, bucket, task.getPathPrefix(), task.getLastPath()); + return listGcsFilesByPrefix(client, bucket, task.getPathPrefix().get(), task.getLastPath()); } /** * Lists GCS filenames filtered by prefix. *