added duplicate detection system

README.md

@@ -13,3 +13,29 @@ PictShare is a selfhostable, open source image, video and text hosting as well a
 - Added URL shortening
 - Added WebP to images (and conversion from jpg,png to webp)
 - Massive code rework. Actually we designed it from the ground up to be more modular and easier to debug
+
+## Status
+
+- [x] Duplicate detection
+- [x] Write permission detection
+
+### Image hosting
+- [x] Upload of images
+- [ ] Resizing
+- [ ] Filters
+- [ ] Gif to mp4 conversion
+
+### Text file hosting
+- [x] Upload of text files
+- [x] Render template for text files
+- [x] Raw data view
+- [x] Downloadable
+
+### URL shortening
+- [ ] Upload of links to shorten
+
+### MP4 hosting
+- [ ] Resizing
+- [x] Upload of videos
+- [x] Automatic conversion if not mobile friendly or wrong encoder used
+- [x] Render template for videos

api/upload.php

@@ -23,12 +23,17 @@ else if(!isFolderWritable(ROOT.DS.'tmp'))
 // check for POST upload
 if ($_FILES['file']["error"] == UPLOAD_ERR_OK)
 {
+    //check for duplicates
+    $sha1 = sha1_file($_FILES['file']["tmp_name"]);
+    $hash = sha1Exists($sha1);
+    if($hash)
+        exit(json_encode(array('status'=>'ok','hash'=>$hash,'url'=>URL.$hash)));
+
     //get the file type
     $type = getTypeOfFile($_FILES['file']["tmp_name"]);
-    //@todo: check for duplicates here
 
     //cross check filetype for controllers
-
+    //
     //image?
     if(in_array($type,(new ImageController)->getRegisteredExtensions()))
     {
@@ -46,7 +51,12 @@ if ($_FILES['file']["error"] == UPLOAD_ERR_OK)
     }
 
     if(!$answer)
-        $answer = array('status'=>'err','reason'=>'Unknown error');
+        $answer = array('status'=>'err','reason'=>'Unsupported filetype');
+
+    if($answer['hash'])
+        addSha1($answer['hash'],$sha1);
 
     echo json_encode($answer);
 }
+else
+    exit(json_encode(array('status'=>'err','reason'=>'Upload error')));

controllers/image/image.controller.php

@@ -28,6 +28,8 @@ class ImageController
                 $res = imagecreatefromjpeg($tmpfile);
                 imagejpeg($res, $tmpfile, (defined('JPEG_COMPRESSION')?JPEG_COMPRESSION:90));
                 $ext = 'jpg';
+
+                $newsha1 = sha1_file($tmpfile);
             break;
 
             default:
@@ -39,6 +41,9 @@ class ImageController
             $hash = getNewHash($ext,6);
         }
 
+        if($newsha1)
+            addSha1($hash,$newsha1);
+
         mkdir(ROOT.DS.'data'.DS.$hash);
 		$file = ROOT.DS.'data'.DS.$hash.DS.$hash;
 		

inc/core.php

@@ -253,6 +253,8 @@ function getTypeOfFile($url)
 	return $type;
 }
 
+function isFolderWritable($dir){return is_writable($dir);}
+
 function getRandomString($length=32, $keyspace = '0123456789abcdefghijklmnopqrstuvwxyz')
 {
     $str = '';
@@ -300,7 +302,26 @@ function getUserIP()
 	return $ip;
 }
 
-function isFolderWritable($dir)
+// checks the list of uploaded files for this hash
+function sha1Exists($sha1)
 {
-    return is_writable($dir);
+    $handle = fopen(ROOT.DS.'data'.DS.'sha1.csv', "r");
+    if ($handle) {
+        while (($line = fgets($handle)) !== false) {
+            if(substr($line,0,40)==$sha1) return trim(substr($line,41));
+        }
+
+        fclose($handle);
+    }
+    return false;
+}
+
+//adds new sha to  the hash list
+function addSha1($hash,$sha1)
+{
+    if(sha1Exists($sha1)) return;
+    $fp = fopen(ROOT.DS.'data'.DS.'sha1.csv','a');
+    fwrite($fp,"$sha1;$hash\n");
+    fclose($fp);
+    return true;
 }

tools/re-encode_mp4.php

@@ -111,6 +111,10 @@ foreach($localfiles as $hash)
     system($cmd);
     if(defined('ALT_FOLDER') && ALT_FOLDER && is_dir(ALT_FOLDER))
         copy($mp4,ALT_FOLDER.DS.$hash);
+
+    //file got a new hash so add that as well
+    addSha1($hash,sha1_file($mp4));
+    
     echo "\tdone\n";
 
 }