# Copyright 2011 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from boto.s3.deletemarker import DeleteMarker
from gslib.bucket_listing_ref import BucketListingRef
from gslib.command import Command
from gslib.command import COMMAND_NAME
from gslib.command import COMMAND_NAME_ALIASES
from gslib.command import CONFIG_REQUIRED
from gslib.command import FILE_URIS_OK
from gslib.command import MAX_ARGS
from gslib.command import MIN_ARGS
from gslib.command import PROVIDER_URIS_OK
from gslib.command import SUPPORTED_SUB_ARGS
from gslib.command import URIS_START_ARG
from gslib.exception import CommandException
from gslib.help_provider import HELP_NAME
from gslib.help_provider import HELP_NAME_ALIASES
from gslib.help_provider import HELP_ONE_LINE_SUMMARY
from gslib.help_provider import HELP_TEXT
from gslib.help_provider import HelpType
from gslib.help_provider import HELP_TYPE
from gslib.plurality_checkable_iterator import PluralityCheckableIterator
from gslib.util import ListingStyle
from gslib.util import MakeHumanReadable
from gslib.util import NO_MAX
from gslib.wildcard_iterator import ContainsWildcard
import boto

_detailed_help_text = ("""
<B>SYNOPSIS</B>
  gsutil ls [-a] [-b] [-l] [-L] [-R] [-p proj_id] uri...


<B>LISTING PROVIDERS, BUCKETS, SUBDIRECTORIES, AND OBJECTS</B>
  If you run gsutil ls without URIs, it lists all of the Google Cloud Storage
  buckets under your default project ID:

    gsutil ls

  (For details about projects, see "gsutil help projects" and also the -p
  option in the OPTIONS section below.)

  If you specify one or more provider URIs, gsutil ls will list buckets at
  each listed provider:

    gsutil ls gs://

  If you specify bucket URIs, gsutil ls will list objects at the top level of
  each bucket, along with the names of each subdirectory. For example:

    gsutil ls gs://bucket

  might produce output like:

    gs://bucket/obj1.htm
    gs://bucket/obj2.htm
    gs://bucket/images1/
    gs://bucket/images2/

  The "/" at the end of the last 2 URIs tells you they are subdirectories,
  which you can list using:

    gsutil ls gs://bucket/images*

  If you specify object URIs, gsutil ls will list the specified objects. For
  example:

    gsutil ls gs://bucket/*.txt

  will list all files whose name matches the above wildcard at the top level
  of the bucket.

  See "gsutil help wildcards" for more details on working with wildcards.


<B>DIRECTORY BY DIRECTORY, FLAT, and RECURSIVE LISTINGS</B>
  Listing a bucket or subdirectory (as illustrated near the end of the previous
  section) only shows the objects and names of subdirectories it contains. You
  can list all objects in a bucket by using the -R option. For example:

    gsutil ls -R gs://bucket

  will list the top-level objects and buckets, then the objects and
  buckets under gs://bucket/images1, then those under gs://bucket/images2, etc.

  If you want to see all objects in the bucket in one "flat" listing use the
  recursive ("**") wildcard, like:

    gsutil ls -R gs://bucket/**

  or, for a flat listing of a subdirectory:

    gsutil ls -R gs://bucket/dir/**


<B>LISTING OBJECT DETAILS</B>
  If you specify the -l option, gsutil will output additional information
  about each matching provider, bucket, subdirectory, or object. For example,

    gsutil ls -l gs://bucket/*.txt

  will print the object size, creation time stamp, and name of each matching
  object, along with the total count and sum of sizes of all matching objects:

       2276224  2012-03-02T19:25:17  gs://bucket/obj1
       3914624  2012-03-02T19:30:27  gs://bucket/obj2
    TOTAL: 2 objects, 6190848 bytes (5.9 MB)

  Note that the total listed in parentheses above is in mebibytes (or gibibytes,
  tebibytes, etc.), which corresponds to the unit of billing measurement for
  Google Cloud Storage.

  You can get a listing of all the objects in the top-level bucket directory
  (along with the total count and sum of sizes) using a command like:

    gsutil ls -l gs://bucket

  To print additional detail about objects and buckets use the gsutil ls -L
  option. For example:

    gsutil ls -L gs://bucket/obj1

  will print something like:

    gs://bucket/obj1:
            Creation Time:      Fri, 02 Mar 2012 19:25:17 GMT
            Size:               2276224
            Cache-Control:      private, max-age=0
            Content-Type:       application/x-executable
            ETag:       5ca6796417570a586723b7344afffc81
            ACL:        <Owner:00b4903a97163d99003117abe64d292561d2b4074fc90ce5c0e35ac45f66ad70, <<UserById: 00b4903a97163d99003117abe64d292561d2b4074fc90ce5c0e35ac45f66ad70>: u'FULL_CONTROL'>>
    TOTAL: 1 objects, 2276224 bytes (2.17 MB)

  Note that the -L option is slower and more costly to use than the -l option,
  because it makes a bucket listing request followed by a HEAD request for
  each individual object (rather than just parsing the information it needs
  out of a single bucket listing, the way the -l option does).

  See also "gsutil help getacl" for getting a more readable version of the ACL.


<B>LISTING BUCKET DETAILS</B>
  If you want to see information about the bucket itself, use the -b
  option. For example:

    gsutil ls -L -b gs://bucket

  will print something like:

    gs://bucket/ :
            24 objects, 29.83 KB
            StorageClass: STANDARD
            LocationConstraint: US
            Versioning enabled: True
            ACL: <Owner:00b4903a9740e42c29800f53bd5a9a62a2f96eb3f64a4313a115df3f3a776bf7, <<GroupById: 00b4903a9740e42c29800f53bd5a9a62a2f96eb3f64a4313a115df3f3a776bf7>: u'FULL_CONTROL'>>
            Default ACL: <>
    TOTAL: 24 objects, 30544 bytes (29.83 KB)


<B>OPTIONS</B>
  -l          Prints long listing (owner, length).

  -L          Prints even more detail than -l. This is a separate option because
              it makes additional service requests (so, takes longer and adds
              requests costs).

  -b          Prints info about the bucket when used with a bucket URI.

  -p proj_id  Specifies the project ID to use for listing buckets.

  -R, -r      Requests a recursive listing.

  -a          Includes non-current object versions / generations in the listing
              (only useful with a versioning-enabled bucket). If combined with
              -l option also prints meta-generation for each listed object.
""")


class LsCommand(Command):
  """Implementation of gsutil ls command."""

  # Command specification (processed by parent class).
  command_spec = {
    # Name of command.
    COMMAND_NAME : 'ls',
    # List of command name aliases.
    COMMAND_NAME_ALIASES : ['dir', 'list'],
    # Min number of args required by this command.
    MIN_ARGS : 0,
    # Max number of args required by this command, or NO_MAX.
    MAX_ARGS : NO_MAX,
    # Getopt-style string specifying acceptable sub args.
    SUPPORTED_SUB_ARGS : 'ablLp:rR',
    # True if file URIs acceptable for this command.
    FILE_URIS_OK : False,
    # True if provider-only URIs acceptable for this command.
    PROVIDER_URIS_OK : True,
    # Index in args of first URI arg.
    URIS_START_ARG : 0,
    # True if must configure gsutil before running command.
    CONFIG_REQUIRED : True,
  }
  help_spec = {
    # Name of command or auxiliary help info for which this help applies.
    HELP_NAME : 'ls',
    # List of help name aliases.
    HELP_NAME_ALIASES : ['dir', 'list'],
    # Type of help:
    HELP_TYPE : HelpType.COMMAND_HELP,
    # One line summary of this help.
    HELP_ONE_LINE_SUMMARY : 'List providers, buckets, or objects',
    # The full help text.
    HELP_TEXT : _detailed_help_text,
  }

  def _PrintBucketInfo(self, bucket_uri, listing_style):
    """Print listing info for given bucket.

    Args:
      bucket_uri: StorageUri being listed.
      listing_style: ListingStyle enum describing type of output desired.

    Returns:
      Tuple (total objects, total bytes) in the bucket.
    """
    bucket_objs = 0
    bucket_bytes = 0
    if listing_style == ListingStyle.SHORT:
      print bucket_uri
    else:
      for obj in self.WildcardIterator(
          bucket_uri.clone_replace_name('**')).IterKeys():
        bucket_objs += 1
        bucket_bytes += obj.size
      if listing_style == ListingStyle.LONG:
        print '%s : %s objects, %s' % (
            bucket_uri, bucket_objs, MakeHumanReadable(bucket_bytes))
      else:  # listing_style == ListingStyle.LONG_LONG:
        location_constraint = bucket_uri.get_location(validate=False,
                                                      headers=self.headers)
        location_output = ''
        if location_constraint:
          location_output = '\n\tLocationConstraint: %s' % location_constraint
        storage_class = bucket_uri.get_storage_class(validate=False,
                                                     headers=self.headers)
        self.proj_id_handler.FillInProjectHeaderIfNeeded(
            'get_acl', bucket_uri, self.headers)
        print('%s :\n\t%d objects, %s\n\tStorageClass: %s%s\n'
              '\tVersioning enabled: %s\n\tACL: %s\n'
              '\tDefault ACL: %s' % (
              bucket_uri, bucket_objs, MakeHumanReadable(bucket_bytes),
              storage_class, location_output,
              bucket_uri.get_versioning_config(),
              bucket_uri.get_acl(False, self.headers),
              bucket_uri.get_def_acl(False, self.headers)))
    return (bucket_objs, bucket_bytes)

  def _UriStrForObj(self, uri, obj):
    """Constructs a URI string for the given object.

    For example if we were iterating gs://*, obj could be an object in one
    of the user's buckets enumerated by the ls command.

    Args:
      uri: base StorageUri being iterated.
      obj: object (Key) being listed.

    Returns:
      URI string.
    """
    version_info = ''
    if self.all_versions:
      if uri.get_provider().name == 'google' and obj.generation:
        version_info = '#%s' % obj.generation
      elif uri.get_provider().name == 'aws' and obj.version_id:
        if isinstance(obj, DeleteMarker):
          version_info = '#<DeleteMarker>' + str(obj.version_id)
        else:
          version_info = '#' + str(obj.version_id)
      else:
        version_info = ''
    return '%s://%s/%s%s' % (uri.scheme, obj.bucket.name, obj.name,
                             version_info)

  def _PrintInfoAboutBucketListingRef(self, bucket_listing_ref, listing_style):
    """Print listing info for given bucket_listing_ref.

    Args:
      bucket_listing_ref: BucketListing being listed.
      listing_style: ListingStyle enum describing type of output desired.

    Returns:
      Tuple (number of objects,
             object length, if listing_style is one of the long listing formats)

    Raises:
      Exception: if calling bug encountered.
    """
    uri = bucket_listing_ref.GetUri()
    obj = bucket_listing_ref.GetKey()
    uri_str = self._UriStrForObj(uri, obj)
    if listing_style == ListingStyle.SHORT:
      print uri_str.encode('utf-8')
      return (1, 0)
    elif listing_style == ListingStyle.LONG:
      # Exclude timestamp fractional secs (example: 2010-08-23T12:46:54.187Z).
      timestamp = obj.last_modified[:19].decode('utf8').encode('ascii')
      if not isinstance(obj, DeleteMarker):
        if self.all_versions:
          print '%10s  %s  %s  meta_generation=%s' % (
              obj.size, timestamp, uri_str.encode('utf-8'), obj.meta_generation)
        else:
          print '%10s  %s  %s' % (obj.size, timestamp, uri_str.encode('utf-8'))
        return (1, obj.size)
      else:
        if self.all_versions:
          print '%10s  %s  %s  meta_generation=%s' % (
              0, timestamp, uri_str.encode('utf-8'), obj.meta_generation)
        else:
          print '%10s  %s  %s' % (0, timestamp, uri_str.encode('utf-8'))
        return (0, 1)
    elif listing_style == ListingStyle.LONG_LONG:
      # Run in a try/except clause so we can continue listings past
      # access-denied errors (which can happen because user may have READ
      # permission on object and thus see the bucket listing data, but lack
      # FULL_CONTROL over individual objects and thus not be able to read
      # their ACLs).
      try:
        print '%s:' % uri_str.encode('utf-8')
        suri = self.suri_builder.StorageUri(uri_str)
        obj = suri.get_key(False)
        print '\tCreation time:\t%s' % obj.last_modified
        if obj.cache_control:
          print '\tCache-Control:\t%s' % obj.cache_control
        if obj.content_disposition:
          print '\tContent-Disposition:\t%s' % obj.content_disposition
        if obj.content_encoding:
          print '\tContent-Encoding:\t%s' % obj.content_encoding
        if obj.content_language:
          print '\tContent-Language:\t%s' % obj.content_language
        print '\tContent-Length:\t%s' % obj.size
        print '\tContent-Type:\t%s' % obj.content_type
        if obj.metadata:
          prefix = uri.get_provider().metadata_prefix
          for name in obj.metadata:
            print '\t%s%s:\t%s' % (prefix, name, obj.metadata[name])
        print '\tETag:\t\t%s' % obj.etag.strip('"\'')
        print '\tACL:\t\t%s' % (suri.get_acl(False, self.headers))
        return (1, obj.size)
      except boto.exception.GSResponseError as e:
        if e.status == 403:
          print ('\tACL:\t\tACCESS DENIED. Note: you need FULL_CONTROL '
                 'permission\n\t\t\ton the object to read its ACL.')
          return (1, obj.size)
        else:
          raise e
    else:
      raise Exception('Unexpected ListingStyle(%s)' % listing_style)

  def _ExpandUriAndPrintInfo(self, uri, listing_style, should_recurse=False):
    """
    Expands wildcards and directories/buckets for uri as needed, and
    calls _PrintInfoAboutBucketListingRef() on each.

    Args:
      uri: StorageUri being listed.
      listing_style: ListingStyle enum describing type of output desired.
      should_recurse: bool indicator of whether to expand recursively.

    Returns:
      Tuple (number of matching objects, number of bytes across these objects).
    """
    # We do a two-level loop, with the outer loop iterating level-by-level from
    # blrs_to_expand, and the inner loop iterating the matches at the current
    # level, printing them, and adding any new subdirs that need expanding to
    # blrs_to_expand (to be picked up in the next outer loop iteration).
    blrs_to_expand = [BucketListingRef(uri)]
    num_objs = 0
    num_bytes = 0
    expanding_top_level = True
    printed_one = False
    num_expanded_blrs = 0
    while len(blrs_to_expand):
      if printed_one:
        print
      blr = blrs_to_expand.pop(0)
      if blr.HasKey():
        blr_iterator = iter([blr])
      elif blr.HasPrefix():
        # Bucket subdir from a previous iteration. Print "header" line only if
        # we're listing more than one subdir (or if it's a recursive listing),
        # to be consistent with the way UNIX ls works.
        if num_expanded_blrs > 1 or should_recurse:
          print '%s:' % blr.GetUriString().encode('utf-8')
          printed_one = True
        blr_iterator = self.WildcardIterator('%s/*' %
                                             blr.GetRStrippedUriString(),
                                             all_versions=self.all_versions)
      elif blr.NamesBucket():
        blr_iterator = self.WildcardIterator('%s*' % blr.GetUriString(),
                                             all_versions=self.all_versions)
      else:
        # This BLR didn't come from a bucket listing. This case happens for
        # BLR's instantiated from a user-provided URI.
        blr_iterator = PluralityCheckableIterator(
            _UriOnlyBlrExpansionIterator(
                self, blr, all_versions=self.all_versions))
        if blr_iterator.is_empty() and not ContainsWildcard(uri):
          raise CommandException('No such object %s' % uri)
      for cur_blr in blr_iterator:
        num_expanded_blrs = num_expanded_blrs + 1
        if cur_blr.HasKey():
          # Object listing.
          (no, nb) = self._PrintInfoAboutBucketListingRef(
              cur_blr, listing_style)
          num_objs += no
          num_bytes += nb
          printed_one = True
        else:
          # Subdir listing. If we're at the top level of a bucket subdir
          # listing don't print the list here (corresponding to how UNIX ls
          # dir just prints its contents, not the name followed by its
          # contents).
          if (expanding_top_level and not uri.names_bucket()) or should_recurse:
            if cur_blr.GetUriString().endswith('//'):
              # Expand gs://bucket// into gs://bucket//* so we don't infinite
              # loop. This case happens when user has uploaded an object whose
              # name begins with a /.
              cur_blr = BucketListingRef(self.suri_builder.StorageUri(
                  '%s*' % cur_blr.GetUriString()), None, None, cur_blr.headers)
            blrs_to_expand.append(cur_blr)
          # Don't include the subdir name in the output if we're doing a
          # recursive listing, as it will be printed as 'subdir:' when we get
          # to the prefix expansion, the next iteration of the main loop.
          else:
            if listing_style == ListingStyle.LONG:
              print '%-33s%s' % (
                  '', cur_blr.GetUriString().encode('utf-8'))
            else:
              print cur_blr.GetUriString().encode('utf-8')
      expanding_top_level = False
    return (num_objs, num_bytes)

  # Command entry point.
  def RunCommand(self):
    got_nomatch_errors = False
    listing_style = ListingStyle.SHORT
    get_bucket_info = False
    self.recursion_requested = False
    self.all_versions = False
    if self.sub_opts:
      for o, a in self.sub_opts:
        if o == '-a':
          self.all_versions = True
        elif o == '-b':
          get_bucket_info = True
        elif o == '-l':
          listing_style = ListingStyle.LONG
        elif o == '-L':
          listing_style = ListingStyle.LONG_LONG
        elif o == '-p':
          self.proj_id_handler.SetProjectId(a)
        elif o == '-r' or o == '-R':
          self.recursion_requested = True

    if not self.args:
      # default to listing all gs buckets
      self.args = ['gs://']

    total_objs = 0
    total_bytes = 0
    for uri_str in self.args:
      uri = self.suri_builder.StorageUri(uri_str)
      self.proj_id_handler.FillInProjectHeaderIfNeeded('ls', uri, self.headers)

      if uri.names_provider():
        # Provider URI: use bucket wildcard to list buckets.
        for uri in self.WildcardIterator('%s://*' % uri.scheme).IterUris():
          (bucket_objs, bucket_bytes) = self._PrintBucketInfo(uri,
                                                              listing_style)
          total_bytes += bucket_bytes
          total_objs += bucket_objs
      elif uri.names_bucket():
        # Bucket URI -> list the object(s) in that bucket.
        if get_bucket_info:
          # ls -b bucket listing request: List info about bucket(s).
          for uri in self.WildcardIterator(uri).IterUris():
            (bucket_objs, bucket_bytes) = self._PrintBucketInfo(uri,
                                                                listing_style)
            total_bytes += bucket_bytes
            total_objs += bucket_objs
        else:
          # Not -b request: List objects in the bucket(s).
          (no, nb) = self._ExpandUriAndPrintInfo(uri, listing_style,
              should_recurse=self.recursion_requested)
          if no == 0 and ContainsWildcard(uri):
            got_nomatch_errors = True
          total_objs += no
          total_bytes += nb
      else:
        # URI names an object or object subdir -> list matching object(s) /
        # subdirs.
        (exp_objs, exp_bytes) = self._ExpandUriAndPrintInfo(uri, listing_style,
            should_recurse=self.recursion_requested)
        if exp_objs == 0 and ContainsWildcard(uri):
          got_nomatch_errors = True
        total_bytes += exp_bytes
        total_objs += exp_objs

    if total_objs and listing_style != ListingStyle.SHORT:
      print ('TOTAL: %d objects, %d bytes (%s)' %
             (total_objs, total_bytes, MakeHumanReadable(float(total_bytes))))
    if got_nomatch_errors:
      raise CommandException('One or more URIs matched no objects.')

    return 0


class _UriOnlyBlrExpansionIterator:
  """
  Iterator that expands a BucketListingRef that contains only a URI (i.e.,
  didn't come from a bucket listing), yielding BucketListingRefs to which it
  expands. This case happens for BLR's instantiated from a user-provided URI.

  Note that we can't use NameExpansionIterator here because it produces an
  iteration over the full object names (e.g., expanding "gs://bucket" to
  "gs://bucket/dir/o1" and "gs://bucket/dir/o2"), while for the ls command
  we need also to see the intermediate directories (like "gs://bucket/dir").
  """
  def __init__(self, command_instance, blr, all_versions=False):
    self.command_instance = command_instance
    self.blr = blr
    self.all_versions=all_versions

  def __iter__(self):
    """
    Args:
      command_instance: calling instance of Command class.
      blr: BucketListingRef to expand.

    Yields:
      List of BucketListingRef to which it expands.
    """
    # Do a delimited wildcard expansion so we get any matches along with
    # whether they are keys or prefixes. That way if bucket contains a key
    # 'abcd' and another key 'abce/x.txt' the expansion will return two BLRs,
    # the first with HasKey()=True and the second with HasPrefix()=True.
    rstripped_versionless_uri_str = self.blr.GetRStrippedUriString()
    if ContainsWildcard(rstripped_versionless_uri_str):
      for blr in self.command_instance.WildcardIterator(
          rstripped_versionless_uri_str, all_versions=self.all_versions):
        yield blr
      return
    # Build a wildcard to expand so CloudWildcardIterator will not just treat it
    # as a key and yield the result without doing a bucket listing.
    for blr in self.command_instance.WildcardIterator(
        rstripped_versionless_uri_str + '*', all_versions=self.all_versions):
      # Find the originally specified BucketListingRef in the expanded list (if
      # present). Don't just use the expanded list, because it would also
      # include objects whose name prefix matches the blr name (because of the
      # wildcard match we did above).  Note that there can be multiple matches,
      # for the case where there's both an object and a subdirectory with the
      # same name.
      if (blr.GetRStrippedUriString()
          == rstripped_versionless_uri_str):
        yield blr