diff --git a/admin/s3-add-inventory.sh b/admin/s3-add-inventory.sh new file mode 100644 index 00000000..680ee118 --- /dev/null +++ b/admin/s3-add-inventory.sh @@ -0,0 +1,24 @@ +#!/bin/sh +SOURCE_BUCKET="bucket1" +DESTINATION_BUCKET="bucket2" +INVENTORY_ID="inventory1" + +aws s3api put-bucket-inventory-configuration \ +--bucket $SOURCE_BUCKET \ +--id $INVENTORY_ID \ +--inventory-configuration \ +'{ + "Schedule": { + "Frequency": "Daily" + }, + "IsEnabled": true, + "Destination": { + "S3BucketDestination": { + "Bucket": "arn:aws:s3:::'$DESTINATION_BUCKET'", + "Format": "CSV" + } + }, + "OptionalFields": ["Size","LastModifiedDate","ETag"], + "IncludedObjectVersions": "Current", + "Id": "'$INVENTORY_ID'" +}' \ No newline at end of file diff --git a/admin/storage_existing_files b/admin/storage_existing_files index a195e347..f1bbd1ff 100755 --- a/admin/storage_existing_files +++ b/admin/storage_existing_files @@ -37,7 +37,7 @@ foreach ($shardIDs as $shardID) { $sql = "SELECT DISTINCT storageFileID FROM storageFileItems"; $ids = Zotero_DB::columnQuery($sql, false, $shardID); - + if (!$ids) continue; $inserted = 0; $origInsertSQL = "INSERT IGNORE INTO storageFilesExisting VALUES "; diff --git a/admin/storage_purge b/admin/storage_purge index 9f36f996..0257adc9 100755 --- a/admin/storage_purge +++ b/admin/storage_purge @@ -14,7 +14,8 @@ define('SET_NAME_DIFF', 'storage:keys:diff'); define('MAX_TIME', time() - (60 * 60 * 24 * 30 * 6)); // 6 months define('DRY_RUN', false); -$startKey = empty($argv[1]) ? false : $argv[1]; +// Gzipped CSV file +$csvFilename = empty($argv[1]) ? false : $argv[1]; $redis = Z_Redis::get(); @@ -26,17 +27,7 @@ if (!$redis->srandmember(SET_NAME_EXISTING)) { $redis->del(SET_NAME_S3); $redis->del(SET_NAME_DIFF); -echo "Listing objects on S3\n"; - -// Get iterator for all files in S3 -$s3 = Z_Core::$AWS->get('s3'); -$options = [ - 'Bucket' => Z_CONFIG::$S3_BUCKET -]; -if ($startKey) { - $options['Marker'] = $startKey; -} -$iterator = $s3->getIterator('ListObjects', $options); +$input = gzopen($csvFilename, 'r'); $deleted = 0; $ignoredS3 = 0; @@ -44,10 +35,10 @@ $ignoredDB = 0; $i = 0; $arr = [SET_NAME_S3]; -foreach ($iterator as $object) { - $key = $object['Key']; - $lastModified = $object['LastModified']; - +while (($row = fgetcsv($input)) !== false) { + $key = $row[1]; + $lastModified = $row[2]; + $date = DateTime::createFromFormat('Y-m-d\TH:i:s.uP', $lastModified); if (!$date) { die("Invalid date '$lastModified'\n"); @@ -84,7 +75,7 @@ if (sizeOf($arr) > 1) { } function deleteFromS3() { - $s3 = Z_Core::$AWS->get('s3'); + $s3 = Z_Core::$AWS->createS3(); $redis = Z_Redis::get(); $deleted = 0; @@ -102,14 +93,10 @@ function deleteFromS3() { // See if file was marked as used recently in the database. The modification time in S3 // isn't updated when an existing file is reused, so this is common. $info = new Zotero_StorageFileInfo; - try { - $info->parseFromS3Key($key); - } - // Ignore other files ('bookmarklet_upload.html') - catch (Exception $e) { - continue; - } + $info->filename = $key; + //$info->zip = ?; + $info = Zotero_Storage::getLocalFileInfo($info); if ($info) { $lastAdded = DateTime::createFromFormat('Y-m-d H:i:s', $info['lastAdded']);