Submit
Path:
~
/
/
usr
/
lib64
/
python3.6
/
site-packages
/
borg
/
__pycache__
/
File Content:
archive.cpython-36.opt-1.pyc
3 �i h � @ s< d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl m Z m Z m Z d dl m Z d dlmZ d dlmZ d dlmZ d dlmZ d d lmZ e� Zd d lmZ d d lmZ d d l m Z d dl m Z m Z d dl!m"Z"m#Z# d dl$T d dl%m&Z&m'Z'm(Z( d dl)m*Z* d dl)m+Z+ d dl)m,Z,m-Z- d dl)m.Z.m/Z/m0Z0 d dl)m1Z1m2Z2m3Z3m4Z4 d dl)m5Z5m6Z6 d dl)m7Z7m8Z8m9Z9m:Z:m;Z; d dl)m<Z<m=Z=m>Z>m?Z? d dl)m@Z@ d dl)mAZA d d l)mBZB d d l)mCZCmDZDmEZE d d l)mFZF d d l)mGZG d d lHmIZImJZJmKZK d d!lLmMZMmNZN d d"lOmPZPmQZQmRZRmSZSmTZTmUZU d d#lVmWZW d d$lXmYZYmZZZ e[ed%�Z\ej]e^ed&d �B Z_e_e^ed'd �B Z`G d(d)� d)�Zad*d+� ZbG d,d-� d-ec�ZdG d.d/� d/ec�ZeG d0d1� d1�Zfef� Zgd2d3� ZhG d4d5� d5�ZiG d6d7� d7�ZjG d8d9� d9ej�ZkG d:d;� d;�Zld<d=� ZmG d>d?� d?�ZnG d@dA� dA�ZoG dBdC� dC�ZpdS )D� N)�contextmanager)�datetime�timezone� timedelta)�partial)�getuser)�BytesIO)�groupby)�get_terminal_size� )� create_logger)�xattr)�Chunker)�ChunkListEntry)� key_factory�UnsupportedPayloadError)� Compressor�CompressionSpec)�*)� ChunkIndex�ChunkIndexEntry�CacheSynchronizer)�Manifest)� hardlinkable)�ChunkIteratorFileWrapper� open_item)�Error�IntegrityError�set_ec)�uid2user�user2uid� gid2group� group2gid)�parse_timestamp� to_localtime)�OutputTimestamp�format_timedelta�format_file_size� file_status�FileSize)� safe_encode� safe_decode�make_path_safe�remove_surrogates)� StableDict)� bin_to_hex)�safe_ns)�ellipsis_truncate�ProgressIndicatorPercent� log_multi)�msgpack)� workarounds)�PathPrefixPattern�FnmatchPattern� IECommand)�Item� ArchiveItem)�acl_get�acl_set� set_flags� get_flags�swidth�hostname)�cache_if_remote)� Repository�LIST_SCAN_LIMIT�linkZO_BINARY� O_NOATIMEc @ sh e Zd Zddd�Zdd� ZdZdd� Zd d � Zd d � Ze d d� �Z e dd� �Z e dd� �Z ddd�Z dS )� StatisticsFc C s( || _ d | _ | _ | _| _d| _d S )Nr )� output_json�osize�csize�usize�nfiles� last_progress)�selfrG � rN � /usr/lib64/python3.6/archive.py�__init__7 s zStatistics.__init__c C s2 | j |7 _ | j|7 _|r.| j|7 _d S )N)rH rI rJ )rM �sizerI �uniquerN rN rO �update< s zStatistics.updatezO{label:15} {stats.osize_fmt:>20s} {stats.csize_fmt:>20s} {stats.usize_fmt:>20s}c C s | j j| dd�S )Nz This archive:)�statsZlabel)�summary�format)rM rN rN rO �__str__D s zStatistics.__str__c C s dj t| �jt| �| d�S )NzF<{cls} object at {hash:#x} ({self.osize}, {self.csize}, {self.usize})>)�cls�hashrM )rV �type�__name__�id)rM rN rN rO �__repr__G s zStatistics.__repr__c C s" t | j�t | j�t | j�| jd�S )N)Z original_sizeZcompressed_sizeZdeduplicated_sizerK )r) rH rI rJ rK )rM rN rN rO �as_dictK s zStatistics.as_dictc C s t | j�S )N)r' rH )rM rN rN rO � osize_fmtS s zStatistics.osize_fmtc C s t | j�S )N)r' rJ )rM rN rN rO � usize_fmtW s zStatistics.usize_fmtc C s t | j�S )N)r' rI )rM rN rN rO � csize_fmt[ s zStatistics.csize_fmtNc C s t j� }|d ks || j |k�r|| _| jrz|sP| j� }t|rD|jnd�|d< ni }|jt j � d|d�� tj |�}d}nrt � \} } |s�dj | �}|r�t|j�nd} | t |� } | dk r�d}| t |� } | dkr�|t | | �7 }nd | }d }t|||p�tjd d � d S ) N� �pathZarchive_progress)�timerZ Zfinished� z={0.osize_fmt} O {0.csize_fmt} C {0.usize_fmt} D {0.nfiles} N � � � � T)�end�file�flush)rd � monotonicrL rG r^ r- rc rS �json�dumpsr rV r? r1 �print�sys�stderr) rM �item�final�stream�dtZnow�data�msgrj �columns�linesrc ZspacerN rN rO � show_progress_ s6 zStatistics.show_progress)F)NFNN)r[ � __module__� __qualname__rP rS rU rW r] r^ �propertyr_ r` ra r{ rN rN rN rO rF 5 s rF c C s t j| �p t j| �p t j| �S )N)�stat�S_ISBLK�S_ISCHR�S_ISFIFO)�moderN rN rO � is_special� s r� c @ s e Zd ZdZdS )� BackupErrorzY Exception raised for non-OSError-based exceptions while accessing backup files. N)r[ r| r} �__doc__rN rN rN rO r� � s r� c @ s e Zd ZdZdd� Zdd� ZdS )� BackupOSErrora� Wrapper for OSError raised while accessing backup files. Borg does different kinds of IO, and IO failures have different consequences. This wrapper represents failures of input file or extraction IO. These are non-critical and are only reported (exit code = 1, warning). Any unwrapped IO error is critical and aborts execution (for example repository IO failure). c C s( || _ || _|j| _|j| _|j| _d S )N)�op�os_error�errno�strerror�filename)rM r� r� rN rN rO rP � s zBackupOSError.__init__c C s$ | j rd| j | jf S t| j�S d S )Nz%s: %s)r� r� �str)rM rN rN rO rW � s zBackupOSError.__str__N)r[ r| r} r� rP rW rN rN rN rO r� � s r� c @ s* e Zd ZdZd dd�Zdd� Zdd� ZdS ) �BackupIOrb c C s || _ | S )N)r� )rM r� rN rN rO �__call__� s zBackupIO.__call__c C s d S )NrN )rM rN rN rO � __enter__� s zBackupIO.__enter__c C s |r t |t�r t| j|�|�d S )N)� issubclass�OSErrorr� r� )rM �exc_typeZexc_valZexc_tbrN rN rO �__exit__� s zBackupIO.__exit__N)rb )r[ r| r} r� r� r� r� rN rN rN rO r� � s r� c c sH dt _x<t �( y t| �}W n tk r. d S X W d Q R X |V qW d S )N�read)� backup_ior� �next� StopIteration)�iteratorrs rN rN rO �backup_io_iter� s r� c @ s( e Zd Zdd� Zd dd�Zd dd�ZdS ) �DownloadPipelinec C s || _ || _d S )N)� repository�key)rM r� r� rN rN rO rP � s zDownloadPipeline.__init__NFc # sb �fdd�}t � }tjdd�}�x<�j|�D �],} |j| � dd� |D �} x&| D ] } d| krPdd� | jD �| _qPW � r�� fd d�| D �} |�rD� o�|�r x�| D ]|} t| j�r�| jd �} | d kr�d| kr�|| j� | jd d �r�|j | j � q�| |kr�|| \} }| d k �r|| � |j | � q�W n$x"| D ]} d| k�r&|| j� �q&W x| D ] } | V �qJW q,W d S )a� Return iterator of items. *ids* is a chunk ID list of an item stream. *filter* is a callable to decide whether an item will be yielded. *preload* preloads the data chunks of every yielded item. Warning: if *preload* is True then all data chunks of every yielded item have to be retrieved, otherwise preloaded chunks will accumulate in RemoteRepository and create a memory leak. c s � j jdd� | D �� d S )Nc S s g | ] }|j �qS rN )r\ )�.0�crN rN rO � <listcomp>� s zBDownloadPipeline.unpack_many.<locals>._preload.<locals>.<listcomp>)r� �preload)�chunks)rM rN rO �_preload� s z.DownloadPipeline.unpack_many.<locals>._preloadF)�use_listc S s g | ]}t |d ��qS ))� internal_dict)r9 )r� rs rN rN rO r� � s z0DownloadPipeline.unpack_many.<locals>.<listcomp>r� c S s g | ] }t |� �qS rN )r )r� �erN rN rO r� � s c s g | ]}� |�r|�qS rN rN )r� rs )�filterrN rO r� � s �sourceN�hardlink_masterT) �setr4 �Unpacker� fetch_many�feedr� r r� �get�addrc )rM �idsr� �partial_extractr� �hardlink_mastersr� Zmasters_preloaded�unpackerrw �itemsrs r� r� �_rN )r� rM rO � unpack_many� s> z DownloadPipeline.unpack_manyc c s8 x2t || jj||d��D ]\}}| jj||�V qW d S )N)� is_preloaded)�zipr� �get_manyr� �decrypt)rM r� r� �id_rw rN rN rO r� s zDownloadPipeline.fetch_many)NFFN)F)r[ r| r} rP r� r� rN rN rN rO r� � s 9r� c @ s> e Zd ZdZefdd�Zdd� Zdd� Zdd d �Zd d � Z dS )� ChunkBufferrg i c C s: t � | _tjdd�| _g | _|| _t| jjf|�� | _ d S )N�surrogateescape)Zunicode_errors) r �bufferr4 ZPacker�packerr� r� r � chunk_seed�chunker)rM r� �chunker_paramsrN rN rO rP s zChunkBuffer.__init__c C s, | j j| jj|j� �� | j� r(| j� d S )N)r� �writer� �packr^ �is_fullrl )rM rs rN rN rO r� s zChunkBuffer.addc C s t �d S )N)�NotImplementedError)rM �chunkrN rN rO � write_chunk s zChunkBuffer.write_chunkFc C s� | j j� dkrd S | j jd� tdd� | jj| j �D ��}| j jd� | j jd� |sbt|�dkrfd nd}x&|d |� D ]}| jj | j |�� qxW |dkr�| j j |d � d S )Nr c s s | ]}t |�V qd S )N)�bytes)r� �srN rN rO � <genexpr> s z$ChunkBuffer.flush.<locals>.<genexpr>r ���r� r� ) r� �tell�seek�listr� �chunkify�truncate�lenr� �appendr� r� )rM rl r� rj r� rN rN rO rl s zChunkBuffer.flushc C s | j j� | jkS )N)r� r� � BUFFER_SIZE)rM rN rN rO r� ' s zChunkBuffer.is_fullNi i � )F) r[ r| r} r� �ITEMS_CHUNKER_PARAMSrP r� r� rl r� rN rN rN rO r� s r� c s( e Zd Zef� fdd� Zdd� Z� ZS )�CacheChunkBufferc s t � j||� || _|| _d S )N)�superrP �cacherT )rM r� r� rT r� )� __class__rN rO rP - s zCacheChunkBuffer.__init__c C s8 | j j| jj|�|| jdd�\}}}| j jjdd� |S )NF)�wait)r� � add_chunkr� �id_hashrT r� �async_response)rM r� r� r� rN rN rO r� 2 s $z CacheChunkBuffer.write_chunk)r[ r| r} r� rP r� � __classcell__rN rN )r� rO r� + s r� c @ s� e Zd ZG dd� de�ZG dd� de�ZG dd� de�Zddd ddddddddedddddfd d �Zd d � Z dd� Z e dd� �Z e dd� �Z e dd� �Ze dd� �Ze dd� �Zdd� Zd d � Zd d � ZdVd d!�ZdWd"d#�ZdXd%d&�Zd'd(� ZdYd)d*�Zd+d,� Zed-d.� �ZdZd0d1�Z d[d2d3�Z d4d5� Z d6d7� Z d\d8d9�Z d:d;� Z!d<d=� Z"d>d?� Z#ed]d@dA��Z$dBdC� Z%dDdE� Z&dFdG� Z'dHdI� Z(dJdK� Z)d^dLdM�Z*dNdO� Z+dPdQ� Z,e-d_dRdS��Z.e-dTdU� �Z/dS )`�Archivec @ s e Zd ZdZdS )zArchive.DoesNotExistzArchive {} does not existN)r[ r| r} r� rN rN rN rO � DoesNotExist: s r� c @ s e Zd ZdZdS )zArchive.AlreadyExistszArchive {} already existsN)r[ r| r} r� rN rN rN rO � AlreadyExists= s r� c @ s e Zd ZdZdS )z+Archive.IncompatibleFilesystemEncodingErrorzrFailed to encode filename "{}" into file system encoding "{}". Consider configuring the LANG environment variable.N)r[ r| r} r� rN rN rN rO �#IncompatibleFilesystemEncodingError@ s r� NFi c C s� t j� | _|| _|| _|| _|| _i | _t|d�| _ || _ || _ || _ d | _ d| _|| _|| _| | _| | _| | _| | _| | _|| _|d kr�tj� }tj� }|| _|| _ || _ |d kr�tj� }|| _ || _ t | j| j�| _!|| _"| j"�rrt#| j| j| j �| _$t%| jj&f|�� | _'||j(k�r$| j)|��tj� | _*d}xrd||�rHd| �pJdf | _+| j+|j(k�rbP |d7 }�q4W n4| jj(j,|�}|d k�r�| j-|��| j.|j/� d | _0d S )N)rG Fr z%s.checkpoint%sz.%drb r )1�os�getcwd�cwdr� r� r� �manifest� hard_linksrF rT r{ �nameZname_in_manifest�comment� tam_verified�checkpoint_interval� numeric_owner�noatime�noctime� nobirthtime� nobsdflags�noacls�noxattrsr �utcnowrd rm r� �start�start_monotonicrj �consider_part_filesr� �pipeline�creater� � items_bufferr r� r� �archivesr� �last_checkpoint�checkpoint_namer� r� �loadr\ �zeros)rM r� r� r� r� r� r� r� r� r� r� r� r� r� r� �progressr� r� r� rj r� Zlog_json�i�inforN rN rO rP C sb zArchive.__init__c C sN | j j|| jj|��}| j j|dd�\}| _}t|d�}|jdkrJtd��|S )NT)�force_tam_not_required)r� r z Unknown archive metadata version) r� r� r� r� �unpack_and_verify_archiver� r: �version� Exception)rM r\ rw �archiver� �metadatarN rN rO � _load_meta| s zArchive._load_metac C sH || _ | j| j �| _dd� | jjD �| j_| jj| _| jjdd�| _d S )Nc S s g | ] }t |��qS rN )r+ )r� �argrN rN rO r� � s z Archive.load.<locals>.<listcomp>r� rb )r\ r r �cmdliner� r� r� )rM r\ rN rN rO r� � s z Archive.loadc C s | j j}t|�S )z,Timestamp of archive creation (start) in UTC)r rd r# )rM �tsrN rN rO r � s z Archive.tsc C s | j jd�p| j j}t|�S )z*Timestamp of archive creation (end) in UTC�time_end)r r� rd r# )rM r rN rN rO �ts_end� s zArchive.ts_endc C s t | j�S )N)r/ r\ )rM rN rN rO �fpr� s z Archive.fprc C s t | j| j �S )N)r& rj r� )rM rN rN rO �duration� s zArchive.durationc C s t | j| j �S )N)r& r r )rM rN rN rO �duration_from_meta� s zArchive.duration_from_metac C s� | j r.| j}| jjtjd�}| jjtjd�}n| j| j�}| j }| j }| j | j t |�t |�|| j� |j� d| jj| j jt id�}| j r�tj|d< n6|j| jj| jj| jj| jjdd�| jjdd�d�� |S ) N)�tzinfoZmax_archive_size)r� r\ r� rj r rT Zlimits� command_liner� rb r� )r r@ �usernamer� r� ) r� rT r� �replacer �utcrj � calc_statsr� r r r� r r% Z total_secondsr^ r� r\ rI � MAX_DATA_SIZErq �argvrS r r r@ r r� )rM rT r� rj r rN rN rO r � s. z Archive.infoc C sB dj | t| jjtjd��t| jjtjd��| jj| j j t d�S )Nz�Archive name: {0.name} Archive fingerprint: {0.fpr} Time (start): {start} Time (end): {end} Duration: {0.duration} Number of files: {0.stats.nfiles} Utilization of max. archive size: {csize_max:.0%} )r )r� rj Z csize_max) rV r% r� r r r rj r� r� r\ rI r )rM rN rN rO rW � s zArchive.__str__c C s d| j S )Nz Archive(%r))r� )rM rN rN rO r] � s zArchive.__repr__c C s$ | j rd|krdS |r ||�S dS )N�partFT)r� )rM rs r� rN rN rO � item_filter� s zArchive.item_filterc # s8 x2�j j�jj|||� �fdd�d�D ] }|V q&W d S )Nc s �j | � �S )N)r )rs )r� rM rN rO �<lambda>� s z$Archive.iter_items.<locals>.<lambda>)r� r� r� r� )r� r� r r� )rM r� r� r� r� rs rN )r� rM rO � iter_items� s zArchive.iter_itemsTc C s* |r| j r| jj |dd� | jj|� d S )Ng�������?)rs rv )r{ rT r� r� )rM rs r{ rN rN rO �add_item� s zArchive.add_itemc C s. | j | j� | jj| j= | jj| j| j� d S )N)�saver� r� r� r� � chunk_decrefr\ rT )rM rN rN rO �write_checkpoint� s zArchive.write_checkpointc C s� |p| j }|| jjkr | j|��| jjdd� ttj� | j d�}|d kr\t j � }|| }n || }|}|| _ || _ d||p~d| jjtjtt� |jt�|jt�| jd� }|j|p�i � t|�}| jj|j� dd�} | jj| �| _ y| j j | j | | j � W nF t k �r> } z(t!| �} d | k�r,t"d | ��n� W Y d d } ~ X nX x| j#j$dd �d k �rZ�qBW | j |jf| jj|< | jj%� | j#j&� | j j&� d S ) NT)rl )Zsecondsr rb ) r r� r� r� r r@ r rd r r� s archive)�contextzMore than allowed put dataz#%s - archive too big (issue #1473)!)r� )'r� r� r� r� r� rl r rd rm r� r r� r� rj r� rq r r@ r ZstrftimeZ ISO_FORMATr� rS r: r� � pack_and_authenticate_metadatar^ r� r\ r� r� rT r r� r r� r� r� �commit) rM r� r� � timestamp�additional_metadatar rj r� r rw �errZerr_msgrN rN rO r � sN z Archive.savec s � �fdd�}t � � t� �}|| j� tt| jj�ddd�}xRt| jj| jj | jj��D ]4\}}|j dd� ||� | j j ||�}|j |� qXW � j�j�d }|j� t� } | jr�|jn |j|j | _| jr�|jn |j|j | _| jr�|jn |j|j | _|| _ | S ) Nc s" �j | }� j| d|j|j� d S )Nr )r� r� rQ rI )r\ �entry)� archive_indexr� rN rO r� s z Archive.calc_stats.<locals>.addz Calculating statistics... %3d%%zarchive.calc_stats)�totalrx �msgidr )�increase� ) r r r\ r2 r� r r� r� r� r� �showr� r� r� Z stats_againstr� �finishrF r� Znum_files_totalsZnum_files_partsrK Z size_totalsZ size_partsrH Z csize_totalsZ csize_partsrI rJ ) rM r� r� �sync�pir\ r� rw Z unique_csizerT rN )r( r� rO r s* $ zArchive.calc_statsc c s� d}d|kr�t jj|f|jjt j�|d � �� }|j|jd |f�\} } | rttrttd�� t j | |� d}W d Q R X n| d k r�| |_ |V | r�|r�tr�d |f||jd�p�|< n d S )NFr� rD T) r� rc �joinr� �split�sepr� �has_linkr� rD r� ) rM �destrs rc �stripped_components� original_pathr� � hardlink_setr� r� Z link_targetrN rN rO �extract_helper8 s $ zArchive.extract_helperr c ? C s� |pi }d|k} |s|r�d|kr�d} x`| j jdd� |jD �dd�D ]@} | rd| jt| �t|j�gd� |rvtjj j | � | t| �7 } qBW |r�tjj j � d |kr�|j } | | kr�t d j| | ���| r�t d ��d S |p�|j}| j}|jjd �r�td��tjj||j�}y6tj|dd�}tj|j��r*tj|� n tj|� W n< tk �r^ | j|tj� �d �Y n t k �rr Y nX dd� }|j }tj |��r<t d�� ||� W d Q R X | j ||||||���r}|�r�d S |�r�| j!d k�r�dd| j"d > | _!t d�� t#|d�}W d Q R X |�� dd� |jD �}x~| j j|dd�D ]j} | �r\| jt| �t|j�gd� t d��6 |�r�| j!j| ��r�|j$t| �d� n |j | � W d Q R X �q8W t d��8 |j%� }} |j&|� |j � | j'|||j(� d � W d Q R X W d Q R X d |k�r |j } | | k�r t d j| | ���| �r.t d ��W d Q R X d S t ��| tj|��r�||� tjj)|��sptj*|� |�r�| j'||� �n4tj+|��r�||� |j,}ytj-||� W n( tk �r� | j|tj� �d �Y nX | j'||dd � n�tj.|��rB||� | j ||||||��&}|�r d S tj/|� | j'||� W d Q R X nxtj0|��sZtj1|��r�||� | j ||||||��.}|�r�d S tj2||j |j3� | j'||� W d Q R X ntd |j ��W d Q R X d S ) a� Extract archive item. :param item: the item to extract :param restore_attrs: restore file attributes :param dry_run: do not write any data :param stdout: write extracted data to stdout :param sparse: write sparse files (chunk-granularity, independent of the original being sparse) :param hardlink_masters: maps paths to (chunks, link_target) for extracting subtrees with hardlinks correctly :param stripped_components: stripped leading path components to correct hard link extraction :param original_path: 'path' key as stored in archive :param pi: ProgressIndicatorPercent (or similar) for file extraction progress (in bytes) �chunks_healthyr� r c S s g | ] }|j �qS rN )r\ )r� r� rN rN rO r� f s z(Archive.extract_item.<locals>.<listcomp>T)r� )r+ r rQ z4Size inconsistency detected: size {}, chunks size {}zDFile has damaged (all-zero) chunks. Try running borg check --repair.N�/�../z!Path should be relative and localF)�follow_symlinksc S s& t jj| �}t jj|�s"t j|� d S )N)r� rc �dirname�exists�makedirs)rc Z parent_dirrN rN rO � make_parent� s z)Archive.extract_item.<locals>.make_parentr@ � r �open�wbc S s g | ] }|j �qS rN )r\ )r� r� rN rN rO r� � s r� Ztruncate_and_attrs)�fd)�symlinkz Unknown archive item type %r)r; r< )4r� r� r� r- r� r- rc rq �stdoutr� r� rl rQ r� rV r� � startswithr r� r1 r �S_ISDIR�st_mode�rmdir�unlink�UnicodeEncodeErrorr� �getfilesystemencodingr� r� �S_ISREGr� r9 r� r� rC r� r� r� � restore_attrs�filenor? �mkdir�S_ISLNKr� rF r� �mkfifor� r� �mknod�rdev)rM rs rP �dry_runrG Zsparser� r6 r7 r0 Zhas_damaged_chunks�item_chunks_sizerw � item_sizer5 rc �strA r� r8 rE r� �posr� rN rN rO � extract_itemR s� " ( zArchive.extract_itemc 2 C s� dt _d }}| js(t|j�}t|j�}|dkr6|jn|}|dkrH|jn|}y*|rbt j |||� nt j |||dd� W n t k r� Y nX |r�t j ||j� nByt j||jdd� W n* ttfk r� |s�t j||j� Y nX |j}d|kr�|j}n|}d|k�rZ|j} y6|�r*t j|d|| fd� nt j|d|| fdd� W n t k �rX Y nX y6|�rxt j|d||fd� nt j|d||fdd� W n t k �r� Y nX | j�s�t||| j� | j�s�|jd i �} x�| j� D ]�\} } y tj |�p�|| | dd� W n� t k �r� } zt| j t j k�r(d }n>| j t j k�r<d }n*| j t j k�rZd t!| �f }n t j"| j �}t#j$d || j%� |� t&t'� W Y dd} ~ X nX �q�W | j( �r�d|k�r�yt)||j*|d� W n t k �r� Y nX dS )zv Restore filesystem attributes on *path* (*fd*) from *item*. Does not access the repository. �attrsNF)r= �atime� birthtime)�ns)r` r= �xattrsztoo big for this filesystemz'xattrs not supported on this filesystemz(no space left on device [xattr len = %d]z*%s: when setting extended attribute %s: %s�bsdflags)rE )+r� r� r� r �userr" �group�uid�gidr� �fchown�chownr� �fchmodr� �chmodr� � SystemError�mtimer^ r_ �utimer� r< r� r� r� r �setxattrr� ZE2BIGZENOTSUPZENOSPCr� r� �logger�warning�decoder Z EXIT_WARNINGr� r= rb )rM rc rs rF rE re rf rl r^ r_ ra �k�vr� Zerr_strrN rN rO rP � sx zArchive.restore_attrsc C sz | j | j�}t|||� | jj|j� dd�}| jj|�}| jj||| j � ||j f| j j | j < | jj| j| j � || _d S )Ns archive)r! )r r\ �setattrr� r" r^ r� r� r� rT rd r� r� r� r )rM r� �valuer rw Znew_idrN rN rO �set_meta2 s zArchive.set_metac C s< || j jkr| j|��| j}|| _| jd|� | j j|= d S )Nr� )r� r� r� r� rv )rM r� ZoldnamerN rN rO �rename< s zArchive.renamec s� G dd� dt �� t� �d����fdd� �� ��fdd�}d��y tjdd �}�jj}tt|�d d d �}x�tt |�j j |���D ]�\}\} } |r�|j |� �j j| | �} |j| � || |� yDx>|D ]6} t| d �} d| kr�x | jD ]\} } }|| |� q�W q�W W q� ttfk �r* �dk�r"� d�Y q�X q�W |�r>|j� W n. tjtjfk �rn �dk�rf� d�Y nX |�j|� �jj�j= x�dd�d k �r��q�W ��r�t j d� t j d� d S )Nc @ s e Zd ZdZdS )z(Archive.delete.<locals>.ChunksIndexErrorzUChunk ID {} missing from chunks index, corrupted chunks index - aborting transaction.N)r[ r| r} r� rN rN rN rO �ChunksIndexErrorE s rx Tc s8 y�j j| d�S tjk r2 �dkr*� d� �S X d S )N)r� r T)r� r� rB �ObjectNotFound)r� )�error�exception_ignored�forcedrM rN rO �fetch_async_responseJ s z,Archive.delete.<locals>.fetch_async_responsec sJ y�j j| |dd� W n$ tk r: t| �}� |��Y n X �dd� d S )NF)r� )r� r �KeyErrorr/ )r\ rT �cid)rx r} rM rN rO r U s z$Archive.delete.<locals>.chunk_decrefF)r� z Decrementing references %3.0f%%zarchive.delete)r) rx r* )r� r� r )r� zAforced deletion succeeded, but the deleted archive was corrupted.z2borg check --repair is required to free all space.)T) r �objectr4 r� r r� r2 r� � enumerater� r� r� r- r� r� r� r9 r� � TypeError� ValueErrorr. ZUnpackExceptionrB ry r\ r� r� r� ro rp )rM rT r� r| r r� Z items_idsr0 r Zitems_idrw rs �chunk_idrQ rI rN )rx rz r{ r} r| rM rO �deleteD sL $ zArchive.deletec C s� t |j|j|jt|j�d�}| js0t|j�|d< | jsDt|j �|d< | j rlt |d�rltt |j d ��|d< | jr�d |d< |d < n t|j�|d< t|j�|d < |S ) N)r� re rf rl r^ �ctime� st_birthtime� � r_ rc rd i ʚ;)�dictrJ �st_uid�st_gidr0 � st_mtime_nsr� � st_atime_nsr� � st_ctime_nsr� �hasattr�intr� r� r r! )rM rZ r] rN rN rO �stat_simple_attrs� s zArchive.stat_simple_attrsc C sz i }t d��H | jri n tj|dd�}| jr0dnt||�}| jsPt|||| j� W d Q R X |rjt |�|d< |rv||d< |S )Nz extended statF)r= r ra rb ) r� r� r Zget_allr� r> r� r; r� r. )rM rZ rc r] ra rb rN rN rO �stat_ext_attrs� s zArchive.stat_ext_attrsc C s | j |�}|j| j||�� |S )N)r� rS r� )rM rZ rc r] rN rN rO � stat_attrs� s zArchive.stat_attrsc c s� t |�}t|d�}d}|o"|jdk}|rT| jj|j|jf�} | d k rP| |_d}nd}||||fV | j|� |r�|| j|j|jf< d S )N)rc Fr �hT) r, r9 �st_nlinkr� r� �st_ino�st_devr� r ) rM rc rZ �statusr Z safe_pathrs r� � hardlinkedr� rN rN rO � create_helper� s zArchive.create_helperc C s< | j ||ddd�� \}}}}|j| j||�� |S Q R X d S )N�dF)r )r� rS r� )rM rc rZ rs r� r� r� rN rN rO � process_dir� s zArchive.process_dirc C s8 | j ||d�� \}}}}|j| j||�� |S Q R X d S )N�f)r� rS r� )rM rc rZ rs r� r� r� rN rN rO � process_fifo� s zArchive.process_fifoc C s@ | j |||��(\}}}}|j|_|j| j||�� |S Q R X d S )N)r� �st_rdevrV rS r� )rM rc rZ Zdev_typers r� r� r� rN rN rO � process_dev� s zArchive.process_devc C s` | j ||ddd��D\}}}}td�� tj|�}W d Q R X ||_|j| j||�� |S Q R X d S )Nr� F)r �readlink)r� r� r� r� r� rS r� )rM rc rZ rs r� r� r� r� rN rN rO �process_symlink� s zArchive.process_symlinkc C st t |j� d�}t|j�}|j|d � |_|jddd� | jd| 7 _||_|d7 }| j|dd� | j� ||fS )N)r� T)�memorize� from_chunksz .borg_part_%dr F)r{ ) r9 r^ r� r� �get_sizerc r r r )rM rs � from_chunkZnumberZlengthrN rN rO �write_part_file� s zArchive.write_part_filec s |s� ��fdd�}g |_ t�dd�r2d|kr2|`d}d}x�|D ]`}|j j||�� �jrj�jj|dd � �jr@tj� �j �jkr@�j |||�\}}tj� �_ q@W |dk�r|j |d � r؈j |||�\}}tj� �_ t � } x"|j D ]} � j | j | | jd � q�W d S ) Nc s. � j �jj| �| �dd�}�jjjdd� |S )NF)r� )r� r� r� r� r� r� )rw � chunk_entry)r� rM rT rN rO �chunk_processor� s z+Archive.chunk_file.<locals>.chunk_processor�recreate_rechunkifyFr: r r g�������?)rs rv )rQ )r� �getattrr: r� r{ rT r� rd rm r� r� rF � chunk_increfr\ rQ ) rM rs r� rT Z chunk_iterr� r� Z part_numberrw Z dummy_statsr� rN )r� rM rT rO � chunk_file� s* zArchive.chunk_filec C s� t |�}|d kr td| ��t|�}|d kr8td| ��ttj� �d }t||d@ dB |||||||d� } tjj} | j | || j t | j j | ��� | jdd� | j jd 7 _| j| � d S ) Nzno such user: %szno such group: %si ʚ;i�� i � ) rc r� re rc rf rd rl r^ r� T)r� r r )r r r" r� rd r9 rq �stdinr� r� rT r� r� r� r� rK r ) rM rc r� r� rc rd re rf �trs rE rN rN rO � process_stdin s& zArchive.process_stdinc s� �j ||d ����\}}}}|j�j|�� t|j�}|rLtjtj|j�B |_| sX|�r�|s�t t j j �j |��} �jj| �} � j| | |�\} } nd } } d \} } d } | d k r�xB| D ]}� j|�s�d}P q�W � �fdd�| D �} d}n | r�dnd}||_| d k �r| |_nxtd�� tj|�}W d Q R X t j|d��&}�j|� �jt�jj ||��� W d Q R X |�s�� j | | |d d� |jD �� �j j d 7 _ |j�j ||�� |j d d � |S Q R X d S )NF�Mc s g | ]}� j |�j��qS rN )r� rT )r� r� )r� rM rN rO r� G s z(Archive.process_file.<locals>.<listcomp>�U�ArC �rbc S s g | ] }|j �qS rN )r\ )r� r� rN rN rO r� W s r T)r� )FN)!r� rS r� r� rJ r �S_IFREG�S_IMODEr� r* r� rc r1 r� r� r� Zfile_known_and_unchangedZ seen_chunkr� r� r� r� �_open_rb�fdopenr� rT r� r� r� Z memorize_filerK r� r� )rM rc rZ r� rs r� r� r� Zis_special_fileZ hashed_pathZ path_hashZknownr� r� r� ZfhrE rN )r� rM rO � process_file+ sD * zArchive.process_filec c s( x"|j D ]}t| ||||d�V qW d S )N)r� )r� r� )r� r� r� r� r� rN rN rO � list_archives] s zArchive.list_archivesc C s� y t j| t�S tk r2 ttkr&� t j| t�S tk r| } z0dtkrj|jtjkrjttkrjt j| t�S � W Y d d }~X nX d S )NZ retry_erofs) r� rC � flags_noatime�PermissionError� flags_normalr� r5 r� ZEROFS)rc �excrN rN rO r� c s zArchive._open_rb)N)NFFN)T)NNNN)TFFFNr NN)FN)FF)NT)N)N)0r[ r| r} r r� r� r� �CHUNKER_PARAMSrP r r� r~ r r r r r r rW r] r r r r r r r r9 r\ rP rv rw r� r� r� r� r� r� r� r� r� r� r� r� r� � staticmethodr� r� rN rN rN rO r� 8 s` 5 . ^ E $2 r� c s� t | �}|dkrdS | d d@ dkr*d}n| d dkr<d}ndS ||krLdS | | d@ d kr^n| | dkrlndS | |d � � t� fdd�|D ��S )z1check if the data <d> looks like a msgpacked dictr F�� � r �� r, �� � �� �� �� Nc 3 s | ]}� j |�V qd S )N)rH )r� �pattern)�key_serializedrN rO r� � s z'valid_msgpacked_dict.<locals>.<genexpr>)r� r� r� )r� �any)r� Zkeys_serializedZd_lenZoffsrN )r� rO �valid_msgpacked_dictv s" r� c sP e Zd ZdZG dd� de�Z� fdd�Zdd� Zdd � Zd d � Z d d � Z � Z S )�RobustUnpackerzCA restartable/robust version of the streaming msgpack unpacker c @ s e Zd ZdZdS )z RobustUnpacker.UnpackerCrashedzraise if unpacker crashedN)r[ r| r} r� rN rN rN rO �UnpackerCrashed� s r� c s> t � j� dd� |D �| _|| _g | _d| _tjtd�| _ d S )Nc S s g | ]}t j|j� ��qS rN )r4 �packb�encode)r� r� rN rN rO r� � s z+RobustUnpacker.__init__.<locals>.<listcomp>F)� object_hook) r� rP � item_keys� validator�_buffered_data�_resyncr4 r� r. � _unpacker)rM r� r� )r� rN rO rP � s zRobustUnpacker.__init__c C s g | _ d| _d S )NT)r� r� )rM rN rN rO �resync� s zRobustUnpacker.resyncc C s$ | j r| jj|� n | jj|� d S )N)r� r� r� r� r� )rM rw rN rN rO r� � s zRobustUnpacker.feedc C s | S )NrN )rM rN rN rO �__iter__� s zRobustUnpacker.__iter__c s� � fdd�}� j r�dj� j�}x�� j r�|s.t�t|� j�sH|dd � }q tjtd�� _ � j j |� y |� }W n � j tfk r� Y nX � j |�r�d� _ |S |dd � }q W n|� S d S )Nc sD y t � j�S ttfk r> } z� jt| ���W Y d d } ~ X nX d S )N)r� r� r� r� r� r� )r& )rM rN rO � unpack_next� s z,RobustUnpacker.__next__.<locals>.unpack_next� r )r� F) r� r1 r� r� r� r� r4 r� r. r� r� r� r� )rM r� rw rs rN )rM rO �__next__� s( zRobustUnpacker.__next__) r[ r| r} r� r r� rP r� r� r� r� r� rN rN )r� rO r� � s r� c @ sZ e Zd Zdd� Zddd�Zd d � Zd d � Zd d� Zdd� Zddd�Z dd� Z ddd�Z dS )�ArchiveCheckerc C s d| _ t� | _d S )NF)� error_foundr� �possibly_superseded)rM rN rN rO rP � s zArchiveChecker.__init__FNr rb c C sN t jd� |dko t|||f� | _|| _|| _| j� | jsLt jd� dS | j |�| _ |rd| j � t j | jkr�t jd� d| _| j� | _nly"t j|t jjf| j d�\| _} W nH tk r� } z,t jd| � d| _| jt j = | j� | _W Y dd} ~ X nX | j|||||d � | j� | j| d � | j�r4t jd � n t jd � | j�pL| j S ) a� Perform a set of checks on 'repository' :param repair: enable repair mode, write updated or corrected data into repository :param archive: only check this archive :param first/last/sort_by: only check this number of first/last archives ordered by sort_by :param glob: only check archives matching this glob :param verify_data: integrity verification of data referenced by archives :param save_space: Repository.commit(save_space) z%Starting archive consistency check...NzJRepository contains no apparent data at all, cannot continue check/repair.Fz Repository manifest not found!T)r� z$Repository manifest is corrupted: %s)r �first�last�sort_by�glob)� save_spacez3Archive consistency check complete, problems found.z6Archive consistency check complete, no problems found.)ro r r� � check_all�repairr� � init_chunksr� rz �make_keyr� � verify_datar � MANIFEST_IDr� �rebuild_manifestr� r� Z OperationZCHECKr �rebuild_refcounts�orphan_chunks_checkr. ) rM r� r� r r� r� r� r� r� r� r� r� rN rN rO �check� s: " zArchiveChecker.checkc C st t t| j�tj d �}t|�| _d}xH| jjt|d�}|s>P |d }tdddd�}x|D ]}|| j|< qZW q(W dS )z8Fetch a list of all object keys from repository g�������?N)�limit�markerr r )�refcountrQ rI r� ) r� r� r� r ZMAX_LOAD_FACTORr� r� rC r )rM Zcapacityr� �resultZ init_entryr� rN rN rO r� s zArchiveChecker.init_chunksc C sz d}xR| j j� D ]D\}}|d7 }|dkr*P |j|�}y t||�S tk rR Y qX qW |dkrfd}nd| }t|��d S )Nr r i� z*make_key: repository has no chunks at all!z4make_key: failed to create the key (tried %d chunks))r� � iteritemsr� r r r )rM r� ZattemptZchunkidr� �cdatarx rN rN rO r� s zArchiveChecker.make_keyc C s� t jd� t| j�}d}d}g }t|dddd�}d }�xf| jjd|d�}|sNP |t|�7 }|d }| jj|�}tt |��} �x | �r�|j � | j d�} y t |�} W n| t jtfk �r } zXd | _|d 7 }t jd t| �| � t| t�r�|j| � | �rtt | ��}| jj|�}W Y d d } ~ X q~X | tjk�r0d n| } y| jj| | � W q~ tk �r� } z0d | _|d 7 }t jd t| �|� |j| � W Y d d }~X q~X q~W q8W |j� ||k�r�t jd � t jd||� |�r�| j�rpt jd� x�|D ]�}y2| jj |�} |tjk�rd n|} | jj| | � W n: tk �rV | j|= | jj |� t j dt|�� Y nX t jdt|�� �q�W n*t jd� x |D ]}t j dt|�� �q�W |�r�t jnt j}|d||� d S )Nz5Starting cryptographic data integrity verification...r zVerifying data %6.2f%%g{�G�z�?zcheck.verify_data)r) rx �stepr* �d )r� r� r Tz chunk %s: %sz chunk %s, integrity error: %szGRepo/Chunks index object count vs. segment files object count mismatch.z:Repo/Chunks index: %d objects != segment files: %d objectszmFound defect chunks. They will be deleted now, so affected files can get repaired now and maybe healed later.zchunk %s deleted.z0chunk %s not deleted, did not consistently fail.z}Found defect chunks. With --repair, they would get deleted, so affected files could get repaired then and maybe healed later.zchunk %s is defect.z`Finished cryptographic data integrity verification, verified %d chunks with %d integrity errors.r� r� ) ro r r� r� r2 r� �scanr� r� �reversedr- �popr� rB ry r r� rz r/ � isinstancer� r r� r� r� r. r� rp r� r� �debug)rM Zchunks_count_indexZchunks_count_segments�errorsZ defect_chunksr0 r� Z chunk_idsZchunk_data_iterZchunk_ids_revdr� Zencrypted_datar& Z _chunk_id�integrity_errorZ defect_chunk�logrN rN rO r� % s| $ zArchiveChecker.verify_datac ) s* t dd� tD ��� � fdd�}tjd� t| j| j�}dd� tD �}tt | j �dd d d �}�x�| j j � D �]�\}}|j � | jj |�}y| jj||�}W n6 tk r� } ztjd | � d | _whW Y dd} ~ X nX t||�s�qhd|kshd|kr�qhytj|�} W n tttfk �r whY nX || �rhy| jj|dd�\} } }W nX tk �r� } z:| j dd�jdd�} tjd| | � tjd� d | _whW Y dd} ~ X nX t| d�} | j} tjd| � | |j k�r d}x(d | |f }||j k�r�P |d7 }�q�W tj d | |� |} || j f|j | < qhW |j � tjd � |S ) z�Rebuild the manifest object if it is missing Iterates through all objects in the repository looking for archive metadata blocks. c s s | ]}|j � V qd S )N)r� )r� r� rN rN rO r� { s z2ArchiveChecker.rebuild_manifest.<locals>.<genexpr>c s t | t�sdS t| �}� j|�S )NF)r� r� r� �issubset)�obj�keys)�required_archive_keysrN rO � valid_archive} s z6ArchiveChecker.rebuild_manifest.<locals>.valid_archivez9Rebuilding missing manifest, this might take some time...c S s g | ]}t j|j� ��qS rN )r4 r� r� )r� r� rN rN rO r� � s z3ArchiveChecker.rebuild_manifest.<locals>.<listcomp>zRebuilding manifest %6.2f%%g{�G�z�?zcheck.rebuild_manifest)r) rx r� r* z Skipping corrupted chunk: %sTNs cmdlines �versionF)r s names <unknown>�asciir z3Archive TAM authentication issue for archive %s: %szMThis archive will *not* be added to the rebuilt manifest! It will be deleted.)r� zFound archive %sr z%s.%dz(Duplicate archive name %s, storing as %szManifest rebuild complete.) � frozensetZREQUIRED_ARCHIVE_KEYSro r r r� r� Z ARCHIVE_KEYSr2 r� r� r� r- r� r� r rz r� r� r4 Zunpackbr� r� r� r rq r: r� r� rp rd r. )rM r r� Zarchive_keys_serializedr0 r� r� r� rw r� r �verifiedr� r� r �new_namerN )r rO r� v sb z ArchiveChecker.rebuild_manifestc 1 s� �j jtjd� �fdd��� �fdd�}d(�fdd� � � ��fdd �}��fd d �}|dkr�|jd �}t|||f�r�jjj||||d �} |r�| r�t j d|� |r�t | �|k r�t j d|t | �� |r�t | �|k r�t j d|t | �� n�jjj|d�} n<y�jj| g} W n( t k �r4 t j d|� d�_dS X t | �} t| dddd�} t�j���d��xRt| �D �]D\} } | j| � t jdj| j| d | �� | j}|�j k�r�t j dt|�� d�_�jj| j= �qh�|� �jj|�}y�jj||�}W nL t k �r@ } z.t j dt|�|� d�_�jj| j= �whW Y dd}~X nX y�jj |d d �\}}}W nT t k �r� } z6t j d | j|� t j d � d�_�jj| j= �whW Y dd}~X nX t |d �}|j dk�r�t d!��d"d#� |j!D �|_!t"�j�}||_#x2||�D ]&}d$|k�r|| j|� |j$|� �q�W |j%dd%� x|j&D ]}�|� �q:W |j |_&�jj'|j(� d&|d'�}�jj)|�}�jj*|�}� |t |�t |�|� || j+f�jj| j< �qhW | j,� W dQ R X dS ))z�Rebuild object reference counts by walking the metadata Missing and/or incorrect data is repaired when detected Nc s, � j j| tddd��jdkr(� jj| � d S )Nr )r� r� r r� r� r� )r� )rM rN rO �mark_as_possibly_superseded� s zEArchiveChecker.rebuild_refcounts.<locals>.mark_as_possibly_supersededc s2 �j j| �}�j j| �}� |t| �t|�|� |S )N)r� r� �encryptr� )r� r� r� )� add_referencerM rN rO � add_callback� s z6ArchiveChecker.rebuild_refcounts.<locals>.add_callbackc sR y� j j| � W n< tk rL td||d�� j | < � jrH� jj| |� Y nX d S )Nr )r� rQ rI )r� Zincrefr~ r r� r� Zput)r� rQ rI r� )rM rN rO r � s z7ArchiveChecker.rebuild_refcounts.<locals>.add_referencec sr �fdd�}d}g }d}d|k}|j }|r0|jn|}|rht|�t|�krhtjdj| |j�� |`d}|}�x~t||�D �]n\} } | \} } } | �j k�r|| | kr�tjdj| |j||| t | ��� d �_ }|| �\} } } }� | | | |� n�tj d j| |j||| t | ��� | \} } } | �j k�r0� | | | � nJtjd j| |j||| t | ��� d �_ }|| �\} } } }� | | | |� nR| | k�r�� | | | � n:tj d j| |j||| t | ��� � | | | � �| d � |j | | | g� || 7 }qvW |�r | �r |j |_|�r(||k�r(tj d j| |j�� |`||_ d |k�rn|j }|jddd�}||k�rntjdj| |j||�� dS )a Verifies that all file chunks are present. Missing file chunks will be replaced with new chunks of the same length containing all zeros. If a previously missing file chunk re-appears, the replacement chunk is replaced by the correct one. c s4 t | �}� jj|�}� jj|�}t|�}|| ||fS )N)r� r� r� r r� )rQ rw r� r� rI )rM rN rO �replacement_chunk� s zWArchiveChecker.rebuild_refcounts.<locals>.verify_file_chunks.<locals>.replacement_chunkr Fr: z0{}: {}: Invalid chunks_healthy metadata removed!z^{}: {}: New missing file chunk detected (Byte {}-{}, Chunk {}). Replacing with all-zero chunk.Tz|{}: {}: Previously missing file chunk is still missing (Byte {}-{}, Chunk {}). It has an all-zero replacement chunk already.zm{}: {}: Missing all-zero replacement chunk detected (Byte {}-{}, Chunk {}). Generating new replacement chunk.zE{}: {}: Healed previously missing file chunk! (Byte {}-{}, Chunk {}).z2{}: {}: Completely healed previously damaged file!rQ )Z compressedr� z<{}: {}: size inconsistency detected: size {}, chunks size {}N)r� r: r� ro rp rV rc r� rz r/ r� r r� rQ r� )� archive_namers r �offsetZ chunk_listZchunks_replacedZhas_chunks_healthyZchunks_currentr: Z chunk_currentZ chunk_healthyr� rQ rI r� rY rX )r r rM rN rO �verify_file_chunks� sf z<ArchiveChecker.rebuild_refcounts.<locals>.verify_file_chunksc 3 s� t dd� �jjD ���t dd� tD ���tdd� �jj�}d� � �fdd�}�fd d �}d d � ����fd d�}d}�xdt| j|�D �]R\}}t|�}|d r�x |D ]}|d||� |d7 }q�W q�|dkr�|j� �xt |�j |��D ]�\}} y\�j j || �} |j | � x>|D ]6} || �\} } | �r2t| d�V n|d| ||� �qW W n� tk �r~ } z|t|�||� W Y dd}~X nN tjk �r� |d||� |j� Y n$ tk �r� |d||� � Y nX |d7 }q�W q�W dS )z�Iterates through all archive items Missing item chunks will be skipped and the msgpack stream will be restarted c s s | ]}|j � V qd S )N)r� )r� r� rN rN rO r� / s zLArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.<genexpr>c s s | ]}|j � V qd S )N)r� )r� r� rN rN rO r� 0 s c S s t | t�od| kS )Ns path)r� r. )rs rN rN rO r 1 s zKArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.<lambda>r c s" � d t | �jk�kr � d7 � � S )N� r )r� r� )r� )�_staterM rN rO �missing_chunk_detector5 s zYArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.missing_chunk_detectorc s, t |�}| d||f 7 } d� _tj| � d S )Nz [chunk: %06d_%s]T)r/ r� ro rz )rx r� Zchunk_nor )rM rN rO �report; s zIArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.reportc S s dj dd� | D ��S )Nz, c s s, | ]$}t |t�r |jd d�nt|�V qdS )r )r� N)r� r� rq r� )r� rr rN rN rO r� B s zdArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.list_keys_safe.<locals>.<genexpr>)r1 )r rN rN rO �list_keys_safeA s zQArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.list_keys_safec sb t | t�sdS | jdd � t| �}�j|�s@dd��| � fS |j� �s^dd�|� � fS d S ) NF�not a dictionarys aclzmissing required keys: zinvalid keys: Trb )Fr )Trb )r� r. r� r� r� )r r )r� r �required_item_keysrN rO � valid_itemD s zMArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.valid_itemr zitem metadata chunk missingr )r� zDDid not get expected metadata dict when unpacking item metadata (%s)NzCUnpacker crashed while unpacking item metadata, trying to resync...z5Exception while decrypting or unpacking item metadata)r r� r� ZREQUIRED_ITEM_KEYSr� r r� r� r� r� r� r� r� r� r9 r r� r� r )r r� r r r r �stater� r� r� rw rs Zvalid�reasonr� )r� rM )r r� r r rO �robust_iterator* sJ z9ArchiveChecker.rebuild_refcounts.<locals>.robust_iterator�,)r� r� r� r� z.--glob-archives %s does not match any archivesz+--first %d archives: only found %d archivesz*--last %d archives: only found %d archives)r� zArchive '%s' not found.TzChecking archives %3.1f%%g�������?zcheck.rebuild_refcounts)r) rx r� r* z Analyzing archive {} ({}/{})r z%Archive metadata block %s is missing!z*Archive metadata block %s is corrupted: %sF)r z3Archive TAM authentication issue for archive %s: %szEThis archive will be *removed* from the manifest! It will be deleted.)r� z Unknown archive metadata versionc S s g | ] }t |��qS rN )r+ )r� r rN rN rO r� � s z4ArchiveChecker.rebuild_refcounts.<locals>.<listcomp>r� )rl s archive)r! �salt)N)-r� r� r r� r2 r� r� r� r� ro rp r� r~ rz r� r2 rA r� r� r- r rV r� r\ r/ r� r� r� r r r: r r r r� r� r� rl r� r" r^ r� r r r. )rM r r� r� r� r� r r r Z archive_infosZ num_archivesr0 r r Z archive_idr� rw r� r r r� rs Zprevious_item_idZnew_archive_idrN )r r r� rM rO r� � s� LH z ArchiveChecker.rebuild_refcountsc C s� | j r�dd� | jj� D �}|| j }|rBtjdjt|��� d| _| j r�|r�tj dt|�t| j�f � x|D ]}| j j |� qnW tj d� n tj d� d S )Nc S s h | ]\}}|j d kr|�qS )r )r� )r� r� r' rN rN rO � <setcomp>� s z5ArchiveChecker.orphan_chunks_check.<locals>.<setcomp>z{} orphaned objects found!Tz1Deleting %d orphaned and %d superseded objects...z.Finished deleting orphaned/superseded objects.z<Orphaned objects check skipped (needs all archives checked).) r� r� r� r� ro rz rV r� r� r� r r� r� )rM ZunusedZorphanedr� rN rN rO r� � s z"ArchiveChecker.orphan_chunks_checkc C s@ | j r<tjd� | jj� tjd� | jj|d� tjd� d S )NzWriting Manifest.z>Committing repo (may take a while, due to compact_segments)...)r� zFinished committing repo.)r� ro r r� r� r� r# )rM r� rN rN rO r. � s zArchiveChecker.finish)FNr r rb NFF)Nr r rb N)F) r[ r| r} rP r� r� r� r� r� r� r� r. rN rN rN rO r� � s *QM {r� c @ s� e Zd ZG dd� de�Zedd� �Zd!dd �Zd"d d �Zd d � Z dd� Z dd� Z dd� Z dd� Z d#dd�Zdd� Zd$dd �Zd d � Zd d � ZdS )%�ArchiveRecreaterc @ s e Zd Zddd�ZdS )z ArchiveRecreater.InterruptedNc C s |pi | _ d S )N)r )rM r rN rN rO rP � s z%ArchiveRecreater.Interrupted.__init__)N)r[ r| r} rP rN rN rN rO � Interrupted� s r c C s | j d�S )Nz .recreate)�endswith)r rN rN rO �is_temporary_archive� s z%ArchiveRecreater.is_temporary_archiveFN� c C s� || _ || _|| _|| _|| _|| _|p*g | _|| _| d k | _| jrPt j d| � | pVt | _ | | _ | | _| pptd�| _t� | _|| _| | _|| _|| _|p�dd� | _| r�d n|| _d S )NzRechunking archives to %sZnonec W s d S )NrN )�argsrN rN rO r � s z+ArchiveRecreater.__init__.<locals>.<lambda>)r� r� r� r� �matcher�exclude_caches�exclude_if_present�keep_exclude_tags� rechunkifyro r� r� r� � recompress�always_recompressr � compressionr� � seen_chunksr$ rW rT r� �print_file_statusr� )rM r� r� r� r� r$ r% r& r' r� r+ r) r* rW rT r� Zfile_status_printerr$ r� rN rN rO rP � s, zArchiveRecreater.__init__c C s| | j |�}| j||�}| js"| jr,| j|� | jj� rR| j rR|j rR|d krRdS | j ||� |d k}| j ||||d� dS )NF)�replace_originalT) � open_archive� create_targetr& r% �matcher_add_tagged_dirsr$ �emptyr) r� � process_itemsr )rM r r� � target_namer �targetr. rN rN rO �recreate� s " zArchiveRecreater.recreatec s | j }|j� � � ri nd }� fdd�}x�|j� D ]�}|j|j�sv| jd|j� ||�r2|jd�|jd�d f||j< q2� r�t|j�r�|jd�|kr�||j \}}} | d kr�||_ |d k r�||_ d d |jf||j < |` n| |_ | j r�| jd|j� q2| j |||� q2W | j�r|jjdd � d S ) Nc s" � o t | j�o | jdd�o d| kS )Nr� Tr� )r r� r� )rs )�target_is_subsetrN rO �item_is_hardlink_master s z?ArchiveRecreater.process_items.<locals>.item_is_hardlink_master�xr� r: r� �-T)rt )r$ r2 r �matchrc r- r� r r� r� r� r: rW � process_itemr� rT r{ ) rM r r5 r$ r� r8 rs r� r: Z new_sourcerN )r7 rO r3 s0 z ArchiveRecreater.process_itemsc C sH d|kr&| j |||� |j jd7 _|j|� | jt|j�|j� d S )Nr� r )�process_chunksrT rK r r- r( r� rc )rM r r5 rs rN rN rO r< 0 s z ArchiveRecreater.process_itemc C sv | j r<|j r<x$|jD ]\}}}| jj||j� qW |jS | j||t|j��}t| j |�}|j || j|j||� d S )N) r) r� r� r� r� rT � iter_chunksr� r r� r� ) rM r r5 rs r� rQ rI �chunk_iteratorr� rN rN rO r= 7 s z ArchiveRecreater.process_chunksc C s� | j j|�}|| jkr&| jj||j�S | j}| jr�| j r�|| jjkr�| j j d | j j |�dd�}t j |�j| j jj|�jkr�d}| jj|||j|dd�}| jj jdd� | jj|j� |S )NF)Z decompress)� overwriter� )r� )r� r� r, r� r� rT r) r* r� r� r� r� r Zdetectr� Z compressorZdecider� r� r� r\ )rM r5 rw r� r@ Z old_chunkr� rN rN rO r� @ s z ArchiveRecreater.chunk_processorc c sP |j jdd� |D ��}|jr8t|�}|jj|�E d H nx|D ] }|V q>W d S )Nc S s g | ]\}}}|�qS rN rN )r� r� r� rN rN rO r� Q s z0ArchiveRecreater.iter_chunks.<locals>.<listcomp>)r� r� r� r r� r� )rM r r5 r� r? rk r� rN rN rO r> P s z ArchiveRecreater.iter_chunksTc C s� | j r d S |d kr |jjdd�}| jr,|j}| jd kr`|jj|jjd�pN|jj|jjtj d�}n|jjtj d�}|j || j|d� |r�|j t � | j d� |j|j� | jr�||_tj� |_ttt|�tt|j�t| j�t� d S )Nr� rb r )rd r r �recreate_cmdline)r rA )r� r$ r% )r� )rW r r� rT r� r$ rd r rq r r r� rF r� rw r� r r� rj r3 ZDASHESr� r� )rM r r5 r� r. Z_startr% rN rN rO r [ s6 zArchiveRecreater.savec s> ���fdd�}�j � g �g �i }�jrhx>|jdd� d�D ]*}tj|j�r:d|kr:d|kr:d||j< q:W x�|j� fd d�d�D ]�}�jr�|j|kr�|||j< tjj |j�\}}|�j kr�|||� q~�jo�|t ko�tj|j�r~d|kr�|n||j }t ||�}|j tt��tkr~|||� q~W � j�tj� � j�tj� dS ) zLAdd excludes to the matcher created by exclude_cache and exclude_if_present.c sH � j r2�jt|jdd�� �jt| d dd�� n�jt| dd�� d S )NF)Z recurse_dirr; )r' r� r6 rc r7 )�dirZtag_item)rM � tag_files� tagged_dirsrN rO �exclude� s z9ArchiveRecreater.matcher_add_tagged_dirs.<locals>.excludec S s t jj| j�tkS )N)r� rc �basename�CACHE_TAG_NAME)rs rN rN rO r � s z:ArchiveRecreater.matcher_add_tagged_dirs.<locals>.<lambda>)r� r� r� Nc s t jj| j�tkp � j| j�S )N)r� rc rF rG r; )rs )r$ rN rO r � s )r$ r% r r rO r� r� rc r� r2 r& rG r r� r� ZCACHE_TAG_CONTENTSr� r8 ZIncludeZExcludeNoRecurse) rM r rE Zcachedir_mastersrs rB Ztag_fileZ content_itemrk rN )r$ rM rC rD rO r1 � s. z(ArchiveRecreater.matcher_add_tagged_dirsc C sZ |p |j d }| j|�}t|jjdg ��}| jo8||jk|_|jrVtj d|pNd|j� |S )zCreate target archive.z .recreater� z Rechunking archive from %s to %sz (unknown)) r� �create_target_archive�tupler r� r( r� r� ro r� )rM r r4 r5 Zsource_chunker_paramsrN rN rO r0 � s z ArchiveRecreater.create_targetc C s, t | j| j| j|d| j| j| j| jd� }|S )NT)r� r� r� r� r� )r� r� r� r� r� r� r� r� )rM r� r5 rN rN rO rH � s z&ArchiveRecreater.create_target_archivec K s" t | j| j| j|fd| ji|��S )Nr� )r� r� r� r� r� )rM r� �kwargsrN rN rO r/ � s z ArchiveRecreater.open_archive) FNFNNFFFFFNNr" )NN)NT)N)r[ r| r} r r r� r! rP r6 r3 r< r= r� r> r r1 r0 rH r/ rN rN rN rO r � s" % (* r )qr� rn r� Zsocketr rq rd � contextlibr r r r � functoolsr Zgetpassr �ior � itertoolsr Zshutilr ro r rb r r� r r� r Z crypto.keyr r �compressr r Z constantsZ hashindexr r r Zhelpersr r r r r r r r r r! r" r# r$ r% r& r' r( r) r* r+ r, r- r. r/ r0 r1 r2 r3 r4 r5 Zpatternsr6 r7 r8 rs r9 r: �platformr; r< r= r> r? r@ ZremoterA r� rB rC r� r4 �O_RDONLYr� r� r� rF r� r r� r� r� r� r� r� r� r� r� r� r� r� r rN rN rN rO �<module> s� L D& D =
Submit
FILE
FOLDER
Name
Size
Permission
Action
__init__.cpython-36.opt-1.pyc
289 bytes
0644
__init__.cpython-36.pyc
719 bytes
0644
__main__.cpython-36.opt-1.pyc
165 bytes
0644
__main__.cpython-36.pyc
165 bytes
0644
_version.cpython-36.opt-1.pyc
133 bytes
0644
_version.cpython-36.pyc
133 bytes
0644
archive.cpython-36.opt-1.pyc
61040 bytes
0644
archive.cpython-36.pyc
61249 bytes
0644
archiver.cpython-36.opt-1.pyc
177490 bytes
0644
archiver.cpython-36.pyc
178018 bytes
0644
cache.cpython-36.opt-1.pyc
40019 bytes
0644
cache.cpython-36.pyc
40279 bytes
0644
constants.cpython-36.opt-1.pyc
1870 bytes
0644
constants.cpython-36.pyc
1915 bytes
0644
fuse.cpython-36.opt-1.pyc
17457 bytes
0644
fuse.cpython-36.pyc
17531 bytes
0644
helpers.cpython-36.opt-1.pyc
81995 bytes
0644
helpers.cpython-36.pyc
82837 bytes
0644
locking.cpython-36.opt-1.pyc
13814 bytes
0644
locking.cpython-36.pyc
13888 bytes
0644
logger.cpython-36.opt-1.pyc
7593 bytes
0644
logger.cpython-36.pyc
7593 bytes
0644
lrucache.cpython-36.opt-1.pyc
1994 bytes
0644
lrucache.cpython-36.pyc
2191 bytes
0644
nanorst.cpython-36.opt-1.pyc
5565 bytes
0644
nanorst.cpython-36.pyc
5669 bytes
0644
patterns.cpython-36.opt-1.pyc
13541 bytes
0644
patterns.cpython-36.pyc
13541 bytes
0644
remote.cpython-36.opt-1.pyc
36814 bytes
0644
remote.cpython-36.pyc
37012 bytes
0644
repository.cpython-36.opt-1.pyc
49853 bytes
0644
repository.cpython-36.pyc
50395 bytes
0644
selftest.cpython-36.opt-1.pyc
3406 bytes
0644
selftest.cpython-36.pyc
3566 bytes
0644
shellpattern.cpython-36.opt-1.pyc
1775 bytes
0644
shellpattern.cpython-36.pyc
1775 bytes
0644
upgrader.cpython-36.opt-1.pyc
12017 bytes
0644
upgrader.cpython-36.pyc
12017 bytes
0644
version.cpython-36.opt-1.pyc
1948 bytes
0644
version.cpython-36.pyc
1948 bytes
0644
xattr.cpython-36.opt-1.pyc
11749 bytes
0644
xattr.cpython-36.pyc
11749 bytes
0644
N4ST4R_ID | Naxtarrr
