sync script added and fixed some controllers

interfaces/storagecontroller.interface.php

@@ -26,6 +26,14 @@ interface StorageController
      */
     function hashExists($hash);
 
+
+    /**
+     * Returns an array of all items in this storage controller
+     * 
+     * @return array
+     */
+    function getItems();
+
     /**
      * If a file does exist in this storage system, then this method should
      * get the file and put it in the default data directory

storage-controllers/altfolder.controller.php

@@ -13,6 +13,20 @@ class AltfolderStorage implements StorageController
 		return file_exists($altname);
     }
 
+    function getItems()
+    {
+        $rii = new RecursiveIteratorIterator(new RecursiveDirectoryIterator(ALT_FOLDER.DS));
+        $files = array(); 
+
+        foreach ($rii as $file) {
+            if ($file->isDir())
+                continue;
+            $files[] = $file->getPathname(); 
+        }
+
+        return $files;
+    }
+
     function pullFile($hash,$location)
     {
         $altname=ALT_FOLDER.DS.$hash;
@@ -25,8 +39,7 @@ class AltfolderStorage implements StorageController
     function pushFile($source,$hash)
     {
         $altname=ALT_FOLDER.DS.$hash;
-        $orig = ROOT.DS.'data'.DS.$hash.DS.$hash;
+		if(!$this->hashExists($hash))
-		if(file_exists($orig) && !$this->hashExists($hash))
 		{
             copy($source,$altname);
             return true;

storage-controllers/s3.controller.php

@@ -36,7 +36,23 @@ class S3Storage implements StorageController
 		if(!$this->s3)$this->connect();
 
 		return $this->s3->doesObjectExist(S3_BUCKET,$hash);
-    }
+	}
+	
+	function getItems()
+	{
+		if(!$this->s3)$this->connect();
+
+		$iterator = $this->s3->getIterator('ListObjects', [
+			'Bucket' => S3_BUCKET
+		]);
+
+		$items = array();
+		foreach ($iterator as $object) {
+			$items[] = $object['Key'];
+		}
+
+		return $items;
+	}
 
     function pullFile($hash,$location)
     {

tools/altfolder_copy.php

@@ -1,83 +0,0 @@
-<?php
-
-/*
-* Alternative folder upload
-* This tool copies all raw images/videos/gifs to the defined ALT_FOLDER location
-* This will create a copy in the location. The location can be a mounted external server like CIFS or sshfs
-* This will allow you to store a backup of your images on some other server
-*
-*/ 
-
-if(php_sapi_name() !== 'cli') exit('This script can only be called via CLI');
-error_reporting(E_ALL & ~E_DEPRECATED & ~E_STRICT & ~E_NOTICE);
-define('DS', DIRECTORY_SEPARATOR);
-define('ROOT', dirname(__FILE__).DS.'..');
-include_once(ROOT.DS.'inc/config.inc.php');
-include_once(ROOT.DS.'inc/core.php');
-require_once(ROOT . DS . 'content-controllers' . DS. 'video'. DS . 'video.controller.php');
-
-if(!defined('ALT_FOLDER') || !ALT_FOLDER)
-die("[X] Error: You should define the ALT_FOLDER config in your inc/config.inc.php first");
-
-if(in_array('sim',$argv))
-{
-    echo "[!!!!] SIMULATION MODE. Nothing will be uploaded [!!!!] \n\n";
-    $sim = true;
-}
-else $sim = false;
-
-
-//gather local data
-echo "[i] Looping through local files\n";
-
-$dir = ROOT.DS.'data'.DS;
-$dh  = opendir($dir);
-$localfiles = array();
-
-$allhashes=0;$allsize=0;
-$skips=0;$skipsize=0;
-$copied=0;$copysize=0;
-$errors=0;$errorsize=0;
-
-while (false !== ($hash = readdir($dh))) {
-    if($hash=='.'||$hash=='..') continue;
-    $img = $dir.$hash.DS.$hash;
-    if(!file_exists($img)) continue;
-    $info = strtolower(pathinfo($img, PATHINFO_EXTENSION));
-    $thissize = filesize($img);
-    $type = getTypeOfFile($img);
-    ++$allhashes;
-    $allsize+=$thissize;
-    if($type)
-    {
-        if(file_exists(ALT_FOLDER.DS.$hash))
-        {
-            echo "  [!] Skipping existing $hash                     \n";
-            ++$skips;
-            $skipsize+=$thissize;
-        }
-        else
-        {
-            ++$copied;
-            $copysize+=$thissize;
-            echo "[i] Copying $hash   to   ".ALT_FOLDER.DS.$hash."                     \r";
-            if($sim===false)
-                copy($img,ALT_FOLDER.DS.$hash);
-        }
-    }
-    else
-    {
-        ++$errors;
-        $errorsize+=$thissize;
-        echo "  [X] ERROR $hash not allowed format: $info                     \n";
-    }
-        
-}
-
-echo "\n[i] Done\n";
-echo "\n----------- STATS ----------\n\n";
-echo "   All files found:\t$allhashes\t".renderSize($allsize)."\n";
-echo "   Copied files:\t$copied\t".renderSize($copysize)."\n";
-echo "   Skipped files:\t$skips\t".renderSize($skipsize)."\n";
-echo "   Erroneous files:\t$errors\t".renderSize($errorsize)."\n";
-echo "\n";

tools/storagecontroller-sync.php

@@ -0,0 +1,109 @@
+<?php
+
+/*
+* Storage controller sync
+* This tool copies syncs local files to storage controllers
+*
+*/ 
+
+if(php_sapi_name() !== 'cli') exit('This script can only be called via CLI');
+error_reporting(E_ALL & ~E_DEPRECATED & ~E_STRICT & ~E_NOTICE);
+define('DS', DIRECTORY_SEPARATOR);
+define('ROOT', dirname(__FILE__).DS.'..');
+include_once(ROOT.DS.'inc/config.inc.php');
+include_once(ROOT.DS.'inc/core.php');
+
+$sc = getStorageControllers();
+$count = 0;
+$controllers = array();
+foreach($sc as $contr)
+{
+    if((new $contr())->isEnabled()===true)
+    {
+        $controllers[] = new $contr();
+    }
+}
+
+if(count($controllers)==0)
+    die("[X] Error: You should define at least one storage controller in your inc/config.inc.php first");
+
+if(in_array('sim',$argv))
+{
+    echo "[!!!!] SIMULATION MODE. Nothing will be uploaded [!!!!] \n\n";
+    $sim = true;
+}
+else $sim = false;
+
+$enc=false;
+if(defined('ENCRYPTION_KEY') && ENCRYPTION_KEY)
+{
+    $enc = new Encryption;
+    echo "[i] Encryption key found. Will encrypt on Storage controllers\n";
+}
+
+
+echo "[i] Looping through local files\n";
+
+$dir = ROOT.DS.'data'.DS;
+$dh  = opendir($dir);
+$localfiles = array();
+
+$allhashes=0;$allsize=0;
+$skips=0;$skipsize=0;
+$copied=0;$copysize=0;
+$errors=0;$errorsize=0;
+$uploaded=0;$uploadsize=0;
+
+while (false !== ($hash = readdir($dh))) {
+    if($hash=='.'||$hash=='..') continue;
+    $img = $dir.$hash.DS.$hash;
+    if(!file_exists($img)) continue;
+    //$info = strtolower(pathinfo($img, PATHINFO_EXTENSION));
+    $thissize = filesize($img);
+    if(!isExistingHash($hash))
+        continue;
+    $allhashes++;
+    $allsize+=$thissize;
+    
+    foreach($controllers as $contr)
+    {
+        if((!$enc && !$contr->hashExists($hash)) || $enc && !$contr->hashExists($hash.'.enc'))
+        {
+            //if($sim!==true)
+            
+            if(defined('ENCRYPTION_KEY') && ENCRYPTION_KEY && !$contr->hashExists($hash.'.enc')) //ok so we got an encryption key which means we'll upload the encrypted file
+            {
+                echo "  [u] Controller '".get_class($contr)."' doesn't have $hash. Encrypting and uploading.. ";
+                $encryptedfile = $img.'.enc';
+                
+                if($sim!==true)
+                {
+                    $enc->encryptFile($img,$encryptedfile,base64_decode(ENCRYPTION_KEY));
+                    $uploadsize+=filesize($encryptedfile);
+                    $contr->pushFile($encryptedfile,$hash.'.enc');  
+                    unlink($encryptedfile);
+                }
+            }
+            else
+            {
+                echo "  [u] Controller '".get_class($contr)."' doesn't have $hash. Uploading unencrypted.. ";
+                if($sim!==true)
+                    $contr->pushFile($img,$hash);
+                $uploadsize+=$thissize;
+            }
+
+            echo "done\n";
+            $uploaded++;
+            
+        }
+    }
+        
+}
+
+echo "\n[i] Done\n";
+echo "\n----------- STATS ----------\n\n";
+echo "   All files found:\t$allhashes\t".renderSize($allsize)."\n";
+echo "   Copied files:\t$copied\t".renderSize($copysize)."\n";
+echo "   Skipped files:\t$skips\t".renderSize($skipsize)."\n";
+echo "   Erroneous files:\t$errors\t".renderSize($errorsize)."\n";
+echo "\n";