o Sb @sUdZddlZddlZddlZddlZddlmZmZmZm Z m Z m Z ddl m Z ddlmZddlmZddlmZddlmZmZmZdd lmZdd lmZdd lmZdd lmZdd l m!Z!m"Z"m#Z#m$Z$m%Z%ddl&m'Z'ej(Z)ej*Z+ej,Z-dZ.dZ/ dZ0edefdefgZ1ee2efe3d<edefdefgZ4ee2efe3d<  d3de2de2de e5de e5def ddZ6de2de2defd d!Z7d"ed#edefd$d%Z8d&e eddfd'd(Z9Gd)d*d*e:Z;Gd+d,d,ej<Z=Gd-d.d.e:Z>Gd/d0d0e:Z?Gd1d2d2eZ@dS)4z.Tools for representing files stored in GridFS.N)AnyIterableListMappingNoReturnOptional)Binary)Int64)ObjectId)SON)CorruptGridFile FileExistsNoFile) ASCENDING) ClientSession) Collection)Cursor)ConfigurationErrorCursorNotFoundDuplicateKeyErrorInvalidOperationOperationFailure)ReadPreference ifiles_idn_C_INDEXfilename uploadDate_F_INDEXF field_name docstring read_only closed_onlyreturncsrdtdtffdd }dtdtdtffdd }|r |d7}nr(d |d f}|s3s3t|||d St||d S) zCreate a GridIn property.selfr%cs:r |js tddkr|jdS|jdS)Nz can only get %r on a closed filelengthr)_closedAttributeError_filegetr&r$r!7/tmp/pip-target-onvjaxws/lib/python/gridfs/grid_file.pygetter?s  z!_grid_in_property..gettervaluecs6|jr|jjd|jdid|ii||j<dSN_idz$set)r(_collfiles update_oner*r&r1r!r.r/setterGs"z!_grid_in_property..setter This attribute is read-only.z%s %szUThis attribute is read-only and can only be read after :meth:`close` has been called.docrproperty)r!r"r#r$r0r9r.r-r/_grid_in_property7s  r?cs*dtdtffdd }|d7}t||dS)zCreate a GridOut property.r&r%cs,|dkr|jdS|jdS)Nr'r) _ensure_filer*r+r,r8r.r/r0^sz"_grid_out_property..getterr:r;r=)r!r"r0r.r8r/_grid_out_property[s rAentitykwargscKs"|jjdd}|jdd|i|S)z;Clear the given database/collection object's type registry.N)Z type_registry codec_optionsr.)rDZ with_options)rBrCZ codecoptsr.r.r/_clear_entity_type_registryjsrEsessioncCs|r |jr tddSdS)Nz3GridFS does not support multi-document transactions)Zin_transactionrrFr.r.r/_disallow_transactionsps rHc @s4eZdZUdZ dSdedeededdfddZd ed ed e ddfd d Z dTddZ dTddZ e de fddZeddddZeed<eddZeeed<eddZeeed<eddZeeed<edd dd!Zeed<ed"d#ddZeed$<ed%d&dd!Zejed'<ed(d)dd!Zeeed(<ejed*<e ed+<dedefd,d-Zded.eddfd/d0Zd1eddfd2d3Z dTd4d5Z!defd6d7Z"d8ede#fd9d:Z$dTd;d<Z%dUd>ede#fd?d@Z&de fdAdBZ'de fdCdDZ(d1eddfdEdFZ)dGe*eddfdHdIZ+de fdJdKZ,dVdLdMZ-dNedOedPedefdQdRZ.dS)WGridInzClass to write data to GridFS.Nroot_collectionrFrCr%cKst|ts td|jjstdt|d|vr |d|d<d|vr+|d|d<t|t j d}| dt |d<| dt |d<t|d |t|d |t|d |jt|d |t|d tt|ddt|ddt|ddt|dddS)aWrite a file to GridFS Application developers should generally not need to instantiate this class directly - instead see the methods provided by :class:`~gridfs.GridFS`. Raises :class:`TypeError` if `root_collection` is not an instance of :class:`~pymongo.collection.Collection`. Any of the file level options specified in the `GridFS Spec `_ may be passed as keyword arguments. Any additional keyword arguments will be set as additional fields on the file document. Valid keyword arguments include: - ``"_id"``: unique ID for this file (default: :class:`~bson.objectid.ObjectId`) - this ``"_id"`` must not have already been used for another file - ``"filename"``: human name for the file - ``"contentType"`` or ``"content_type"``: valid mime-type for the file - ``"chunkSize"`` or ``"chunk_size"``: size of each of the chunks, in bytes (default: 255 kb) - ``"encoding"``: encoding used for this file. Any :class:`str` that is written to the file will be converted to :class:`bytes`. :Parameters: - `root_collection`: root collection to write to - `session` (optional): a :class:`~pymongo.client_session.ClientSession` to use for all commands - `**kwargs: Any` (optional): file level options (see above) .. versionchanged:: 4.0 Removed the `disable_md5` parameter. See :ref:`removed-gridfs-checksum` for details. .. versionchanged:: 3.7 Added the `disable_md5` parameter. .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.0 `root_collection` must use an acknowledged :attr:`~pymongo.collection.Collection.write_concern` 1root_collection must be an instance of Collectionz3root_collection must use acknowledged write_concern content_type contentType chunk_size chunkSize)Zread_preferencer3_sessionr4_chunksr*_buffer _positionr _chunk_numberr(F_ensured_indexN) isinstancer TypeErrorZ write_concernZ acknowledgedrrHpoprErZPRIMARYr+r DEFAULT_CHUNK_SIZEobject __setattr__chunksioBytesIO)r&rJrFrCZcollr.r.r/__init__xs* 6zGridIn.__init__ collection index_keyuniquecCsz|jddi|jd}|dur9zdd|j|jdD}Wn ty'g}Ynw||vr;|j|||jddSdSdS)Nr3)Z projectionrFcSsg|]}|dqS)keyr.).0Z index_specr.r.r/ sz)GridIn.__create_index..rG)rbrF)find_onerPZ list_indexesrZ create_indexitems)r&r`rarbr<Z index_keysr.r.r/Z__create_indexs  zGridIn.__create_indexcCsPt|ds&t|j||jjtd||jjt dt |dddSdS)NrUFT) rZ__getattribute__rHrP_GridIn__create_indexr4r5r r\rr[r,r.r.r/Z__ensure_indexess  zGridIn.__ensure_indexescCsN|jjjd|jdi|jd|jjjd|jdi|jdt|dddS)z>Remove all chunks/files that may have been uploaded and close.rr3rGr(TN) r4r\Z delete_manyr*rPr5Z delete_onerZr[r,r.r.r/abortsz GridIn.abortcC|jS)zIs this file closed?)r(r,r.r.r/closedsz GridIn.closedr3"The ``'_id'`` value for this file.T)r#rName of this file.Alias for `filename`.namerMMime-type for this file.rLr'Length (in bytes) of this file.)r$rOChunk size for this file.rNrz!Date that this file was uploaded. upload_datemd5;MD5 of the contents of this file if an md5 sum was created.rRr(cCs ||jvr |j|Std|)Nz#GridIn object has no attribute '%s')r*r)r&rqr.r.r/ __getattr__s   zGridIn.__getattr__r1cCsb||jvs ||jjvrt|||dS||j|<|jr/|jjd|jdid||iidSdSr2) __dict__ __class__rZr[r*r(r4r5r6)r&rqr1r.r.r/r[s  &zGridIn.__setattr__datacCs||sdSt||jksJ|jd|jt|d}z |jj||jdWnt y9| |jdYnw|jd7_|j t|7_ dS)zFlush `data` to a chunk.Nr3)rrr|rGrc) _GridIn__ensure_indexeslenrNr*rTrrQ insert_onerPr_raise_file_existsrS)r&r|chunkr.r.r/Z __flush_data s zGridIn.__flush_datacCs(||j|jt|_dS)z)Flush the buffer contents out to a chunk.N)_GridIn__flush_datarRgetvaluecloser]r^r,r.r.r/Z__flush_buffers zGridIn.__flush_buffercCsbz |t|j|jd<tj|jd<|jjj|j|j dWSt y0| |j YdSw)zFlush the file to the database.r'rrGN) _GridIn__flush_bufferr rSr*datetimeutcnowr4r5rrPrrr3r,r.r.r/Z__flush!s zGridIn.__flushfile_idcCs td|)z3Raise a FileExists exception for the given file_id.zfile with _id %r already exists)r )r&rr.r.r/r-s zGridIn._raise_file_existscCs$|js|t|dddSdS)zFlush the file and close it. A closed file cannot be written any more. Calling :meth:`close` more than once is allowed. r(TN)r(_GridIn__flushrZr[r,r.r.r/r1sz GridIn.closesizecC td)Nreadr]UnsupportedOperationr&rr.r.r/r; z GridIn.readcCdSNFr.r,r.r.r/readable>zGridIn.readablecCrrr.r,r.r.r/seekableArzGridIn.seekablec Cs>|jrtdz|j}Wn3ty?t|ttfstdt|tr7z||j }Wn ty6tdwt |j}Ynw|j dkrv|j|j }|rrz||}Wn tyc|w|j |t||krrdS|||j}|rt||jkr||||j}|rt||jks|j |dS)a{Write data to the file. There is no return value. `data` can be either a string of bytes or a file-like object (implementing :meth:`read`). If the file has an :attr:`encoding` attribute, `data` can also be a :class:`str` instance, which will be encoded as :attr:`encoding` before being written. Due to buffering, the data may not actually be written to the database until the :meth:`close` method is called. Raises :class:`ValueError` if this file is already closed. Raises :class:`TypeError` if `data` is not an instance of :class:`bytes`, a file-like object, or an instance of :class:`str`. Unicode data is only allowed if the file has an :attr:`encoding` attribute. :Parameters: - `data`: string of bytes or file-like object to be written to the file zcannot write to a closed filez+can only write strings or file-like objectsz7must specify an encoding for file in order to write strrN)r( ValueErrorrr)rVstrbytesrWencodeencodingr]r^rRtellrN BaseExceptionrkwriter~rr)r&r|rspaceZto_writer.r.r/rDsD            z GridIn.writesequencecCs|D]}||qdS)zSWrite a sequence of strings to the file. Does not add seperators. N)r)r&rliner.r.r/ writelines}s zGridIn.writelinescCrNTr.r,r.r.r/ writeablerzGridIn.writeablecC|S)z)Support for the context manager protocol.r.r,r.r.r/ __enter__zGridIn.__enter__exc_typeexc_valexc_tbcCs&|dur |dSt|dddS)zSupport for the context manager protocol. Close the file if no exceptions occur and allow exceptions to propagate. Nr(TF)rrZr[r&rrrr.r.r/__exit__s zGridIn.__exit__Nr%Nr)r%rI)/__name__ __module__ __qualname____doc__rrrrr_boolrjr}rkr>rmr?r3__annotations__rrrqrLr'intrNrurrvr]r^ryr[rrrrrrrrrrrrrrrr.r.r.r/rIus\  R     9 rIc szeZdZUdZ   dVdedeedeedeeddf fdd Z e d d Z ee d <e d d Z ee d <e d dZee d<e ddZeee d<e ddZee d<e ddZee d<e ddZeje d<e ddZeeee d<e ddZeeeefe d<e dd Zeee d<ee d!<ee d"<dWd#d$Zdedefd%d&Zdefd'd(Zdefd)d*ZdXd,edefd-d.Z dXd,edefd/d0Z!defd1d2Z"e#fd3ed4edefd5d6Z$defd7d8Z%dYd9d:Z&dWfd;d< Z'd=ede(fd>d?Z)d@ede(fdAdBZ*defdCdDZ+dYdEdFZ,dGedHedIedefdJdKZ-de(fdLdMZ.dWdNdOZ/defdPdQZ0dZd,eede(fdRdSZ1dWdTdUZ2Z3S)[GridOutz!Class to read data out of GridFS.NrJr file_documentrFr%csdt|ts tdt|t|}t|j|_|j |_ ||_ t |_ d|_d|_||_||_dS)a Read a file from GridFS Application developers should generally not need to instantiate this class directly - instead see the methods provided by :class:`~gridfs.GridFS`. Either `file_id` or `file_document` must be specified, `file_document` will be given priority if present. Raises :class:`TypeError` if `root_collection` is not an instance of :class:`~pymongo.collection.Collection`. :Parameters: - `root_collection`: root collection to read from - `file_id` (optional): value of ``"_id"`` for the file to read - `file_document` (optional): file document from `root_collection.files` - `session` (optional): a :class:`~pymongo.client_session.ClientSession` to use for all commands .. versionchanged:: 3.8 For better performance and to better follow the GridFS spec, :class:`GridOut` now uses a single cursor to read all the chunks in the file. .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.0 Creating a GridOut does not immediately retrieve the file metadata from the server. Metadata is fetched when first needed. rKNr)rVrrWrHrEsuperr_r\_GridOut__chunksr5_GridOut__files_GridOut__file_idEMPTY_GridOut__buffer_GridOut__chunk_iter_GridOut__positionr*rP)r&rJrrrFr{r.r/r_s '  zGridOut.__init__r3rnrrorprqrMrrrLr'rsrOrtrNrz'Date that this file was first uploaded.rualiaseszList of aliases for this file.metadatazMetadata attached to this file.rvrwr*rcCsL|js"t|j|jjd|ji|jd|_|js$td|j|jfdSdS)Nr3rGz+no file in gridfs collection %r with _id %r)r*rHrPrrgrrr,r.r.r/r@s zGridOut._ensure_filecCs(|||jvr|j|Std|)Nz$GridOut object has no attribute '%s')r@r*r)rxr.r.r/rys   zGridOut.__getattr__cCrrr.r,r.r.r/rrzGridOut.readablecCst|j}t}t|j}|dkr|j}n6|jt|jkrJt||j|}|jdur4t||j |j ||_|j }|d|j|d}|sJt d|jt|7_t|_|S)z{Reads a chunk at a time. If the current position is within a chunk the remainder of the chunk is returned. rNr|ztruncated chunk) r~rrrrNrr'r_GridOutChunkIteratorrrPnextr )r&received chunk_datarNZ chunk_numberrr.r.r/ readchunks$     zGridOut.readchunkrrcCs|t|j|j}|dks||kr|}|dkrtSd}t}||kr9|}|t|7}| |||ks&||krQ|j rQz|j Wn t yPYnw|j||8_| |||_| d||S)aRead at most `size` bytes from the file (less if there isn't enough data). The bytes are returned as an instance of :class:`str` (:class:`bytes` in python 3). If `size` is negative or omitted all data is read. :Parameters: - `size` (optional): the number of bytes to read .. versionchanged:: 3.8 This method now only checks for extra chunks after reading the entire file. Previously, this method would check for extra chunks on every call. r)r@rr'rrr]r^rr~rrr StopIterationseekrr)r&r remainderrr|rr.r.r/rs0       z GridOut.readcCst|j|j}|dks||kr|}|dkrtSd}t}||krK|}|td|}|dkr7||d}|t |7}| ||dkrGn||ks"|j||8_| || |_ | d| |S)zRead one line or up to `size` bytes from the file. :Parameters: - `size` (optional): the maximum number of bytes to read rrrc)rr'rrr]r^rfindNEWLNr~rrrr)r&rrrr|rposr.r.r/readlineFs,       zGridOut.readlinecCrl)z)Return the current position of this file.)rr,r.r.r/rhsz GridOut.tellrwhencecCs|tkr|}n|tkr|j|}n|tkrt|j|}ntdd|dkr+tdd||jkr2|S||_t|_|j rC|j d|_ |S)agSet the current position of this file. :Parameters: - `pos`: the position (or offset if using relative positioning) to seek to - `whence` (optional): where to seek from. :attr:`os.SEEK_SET` (``0``) for absolute file positioning, :attr:`os.SEEK_CUR` (``1``) to seek relative to the current position, :attr:`os.SEEK_END` (``2``) to seek relative to the file's end. .. versionchanged:: 4.1 The method now returns the new position in the file, to conform to the behavior of :meth:`io.IOBase.seek`. zInvalid value for `whence`rz*Invalid value for `pos` - must be positiveN) _SEEK_SET _SEEK_CURr _SEEK_ENDrr'IOErrorrrrr)r&rrnew_posr.r.r/rls"     z GridOut.seekcCrrr.r,r.r.r/rrzGridOut.seekablecCr)a-Return an iterator over all of this file's data. The iterator will return lines (delimited by ``b'\n'``) of :class:`bytes`. This can be useful when serving files using a webserver that handles such an iterator efficiently. .. versionchanged:: 3.8 The iterator now raises :class:`CorruptGridFile` when encountering any truncated, missing, or extra chunk in a file. The previous behavior was to only raise :class:`CorruptGridFile` on a missing chunk. .. versionchanged:: 4.0 The iterator now iterates over *lines* in the file, instead of chunks, to conform to the base class :py:class:`io.IOBase`. Use :meth:`GridOut.readchunk` to read chunk by chunk instead of line by line. r.r,r.r.r/__iter__szGridOut.__iter__cs$|jr |jd|_tdS)z(Make GridOut more generically file-like.N)rrrr,rr.r/rs z GridOut.closer1cCr)Nrrr7r.r.r/rrz GridOut.writelinescCr)Nrr)r&rr.r.r/rrzGridOut.writelinescCrrr.r,r.r.r/writablerzGridOut.writablecCr)cMakes it possible to use :class:`GridOut` files with the context manager protocol. r.r,r.r.r/rszGridOut.__enter__rrrcCs |dS)rF)rrr.r.r/rszGridOut.__exit__cCr)Nfilenorr,r.r.r/rrzGridOut.filenocCdSrr.r,r.r.r/flushrz GridOut.flushcCrrr.r,r.r.r/isattyrzGridOut.isattycCr)Ntruncaterrr.r.r/rs zGridOut.truncatecCrrr.r,r.r.r/__del__rzGridOut.__del__)NNNrr)r%rr)4rrrrrrrrrr_rAr3rrrrqrLr'rNrurrrrrrvr@ryrrrrrrrrrrrrrrrrrrrrrrr __classcell__r.r.rr/rsn 8  ."'   rc @seZdZUdZdededeededdf dd Z ee e d <d e de fd d Z dddZdddZdeeeffddZdeeeffddZeZdddZdS)rzIterates over a file's chunks using a single cursor. Raises CorruptGridFile when encountering any truncated, missing, or extra chunk in a file. grid_outr\rF next_chunkr%NcCsT|j|_t|j|_t|j|_||_||_||_t t |j|j|_ d|_ dSr)r3rrN _chunk_sizer'_lengthrQrP _next_chunkmathceilfloat _num_chunks_cursor)r&rr\rFrr.r.r/r_s   z_GridOutChunkIterator.__init__rchunk_ncCs*||jdkr |jS|j|j|jdS)Nrc)rrr)r&rr.r.r/expected_chunk_lengthsz+_GridOutChunkIterator.expected_chunk_lengthcCrrr.r,r.r.r/rrz_GridOutChunkIterator.__iter__cCsHd|ji}|jdkrd|ji|d<t|j|jj|dg|jd|_dS)Nrrz$gter)rrc)sortrF)r3rrHrPrQrr)r&filterr.r.r/_create_cursors   z$_GridOutChunkIterator._create_cursorcCs\|jdur||jdusJz|jWSty-|j||jYSw)aReturn the next chunk and retry once on CursorNotFound. We retry on CursorNotFound to maintain backwards compatibility in cases where two calls to read occur more than 10 minutes apart (the server's default cursor timeout). N)rrrrrr,r.r.r/_next_with_retrys    z&_GridOutChunkIterator._next_with_retrycCsz|}Wnty|j|jkrtd|jw|d|jkr2|td|j|df|d|jkrNt|drN|td|j|df||d}t|d|krp|td|d|t|df|jd7_|S)Nz no chunk #%drz;Missing chunk: expected chunk #%d but found chunk with n=%dr|z?Extra chunk found: expected %d chunks but found chunk with n=%dzRtruncated chunk #%d: expected chunk length to be %d but found chunk with length %drc)rrrrr rr~r)r&rZexpected_lengthr.r.r/rs@      z_GridOutChunkIterator.nextcCs|jr |jd|_dSdSr)rrr,r.r.r/r<s  z_GridOutChunkIterator.close)r%rr)rrrrrrrrrr_rrrrrrrrrr__next__rr.r.r.r/rs*    #rc@s>eZdZdededefddZd ddZdefd d Z e Z d S) GridOutIteratorrr\rFcCst|||d|_dS)Nr)r_GridOutIterator__chunk_iter)r&rr\rFr.r.r/r_CszGridOutIterator.__init__r%cCrrr.r,r.r.r/rFrzGridOutIterator.__iter__cCs|j}t|dS)Nr|)rrr)r&rr.r.r/rIs  zGridOutIterator.nextN)r%r) rrrrrrr_rrrrr.r.r.r/rBs  rcseZdZdZ       ddedeeeefde de d e d eed e d ee d dffdd Z d e ffdd ZeZdeded efddZdeded efddZd e d dfddZZS) GridOutCursorzA cursor / iterator for returning GridOut objects as the result of an arbitrary query against the GridFS files collection. NrFr`rskiplimitno_cursor_timeoutr batch_sizerFr%c s<t|t|}||_tt|j|j|||||||ddS)ayCreate a new cursor, similar to the normal :class:`~pymongo.cursor.Cursor`. Should not be called directly by application developers - see the :class:`~gridfs.GridFS` method :meth:`~gridfs.GridFS.find` instead. .. versionadded 2.7 .. seealso:: The MongoDB documentation on `cursors `_. )rrrrrrFN)rHrE_GridOutCursor__root_collectionrrr_r5) r&r`rrrrrrrFrr.r/r_Us  zGridOutCursor.__init__cs*t|jtt|}t|j||jdS)z$Get next GridOut object from cursor.)rrF)rHrFrrrrr)r&Z next_filerr.r/r{s zGridOutCursor.nextargsrCcOtdNz'Method does not exist for GridOutCursorNotImplementedErrorr&rrCr.r.r/ add_optionzGridOutCursor.add_optioncOrrrrr.r.r/ remove_optionrzGridOutCursor.remove_optioncCst|j|dS)zACreates an empty GridOutCursor for information to be copied into.rG)rr)r&rFr.r.r/ _clone_baseszGridOutCursor._clone_base)NrrFNrN)rrrrrrrrrrrrr_rrrrrrrrr.r.rr/rPsB  &r)FF)Arrr]rostypingrrrrrrZ bson.binaryrZ bson.int64r Z bson.objectidr Zbson.sonr Z gridfs.errorsr r rZpymongorZpymongo.client_sessionrZpymongo.collectionrZpymongo.cursorrZpymongo.errorsrrrrrZpymongo.read_preferencesrSEEK_SETrSEEK_CURrSEEK_ENDrrrrYrrrr rr?rArErHrZrIIOBaserrrrr.r.r.r/sf          $$ $)Dd