Changeset 287

Timestamp:

01/26/10 12:24:06 (10 years ago)

Author:

sander

Message:

Change on-disk file layout to support nested sets of documents

Location:

trunk/server/www/app

Files:

• controllers/jobs_controller.php (modified) (3 diffs)
• controllers/requests_controller.php (modified) (4 diffs)
• controllers/results_controller.php (modified) (1 diff)
• models/behaviors/file.php (modified) (12 diffs)
• models/request.php (modified) (8 diffs)
• models/result.php (modified) (1 diff)
• tmp/zips (added)

trunk/server/www/app/controllers/jobs_controller.php

@@ -349 +349 @@
                 $job = array_shift($jobs);
 
-                $file = $this->Request->Behaviors->File->settings['directory'] . DS . $job['Request']['path'];
+                $this->Request->id = $job['Request']['id'];
+                $file = $this->Request->getPath();
                 if (!file_exists($file) || !is_readable($file)) {
                         // Something went wrong. Expire the entire request
-                        $this->Request->id = $job['Request']['id'];
                         $this->Request->saveField('expire', date('Y-m-d H:i:s', time() -1));
                         return array('faultCode' => 2, 'faultString' => 'Job document could not be read. Please poll again.');
@@ -445 +445 @@
                 $job = $this->Job->find('first', array(
                         'conditions' => array('Job.id' => $job_id),
-                        'contain' => array('Request', 'Request.Format')
+                        'contain' => array(
+                                'Request',
+                                'Request.Format',
+                                'Application',
+                                'Platform',
+                        )
                 ));
 
@@ -475 +480 @@
                 }
 
+                // Find out where to store the result
+                $path  = $job['Request']['root'] . DS;
+                $path .= Inflector::slug($job['Application']['name']) . '_';
+                $path .= Inflector::slug($job['Job']['version']) . '_';
+                $path .= Inflector::slug($job['Platform']['name']);
+
                 // Create the Result
                 $this->Job->Result->create();
                 $this->Job->Result->set(array(
                         'format_id'  => $format['Format']['id'],
-                        'factory_id' => $factory['Factory']['id']
+                        'factory_id' => $factory['Factory']['id'],
+                        'path' => $path
                 ));
 

trunk/server/www/app/controllers/requests_controller.php

@@ -204 +204 @@
                         }
 
-                        // Create a new request
+                        // Set some extra data
+                        $this->data['Request']['user_id']    = (string) $this->AuthCert->user('id');
+                        $this->data['Request']['ip_address'] = inet_ptod($this->RequestHandler->getClientIP());
+                        $this->data['Request']['expire']     = date('Y-m-d H:i:s', time() + Configure::read('Request.expire'));
+                        $this->data['Request']['state']      = Request::STATE_PREPROCESSOR_QUEUED;
+
+
+                        // Create the request
                         $this->Request->create();
-                        $this->data['Request']['user_id'] = (string) $this->AuthCert->user('id');
-                        $this->data['Request']['ip_address'] = inet_ptod($this->RequestHandler->getClientIP());
+                        if (!$this->Request->save($this->data)) {
+                                $this->Session->setFlash(__('The request could not be saved', true));
+                                $this->redirect(array('action'=>'add'));
+                        }
+
+                        // Set root and path based on the ID
+                        $this->Request->read();
+                        $this->Request->set(array(
+                                'root' => 'requests' . DS . $this->Request->id,
+                                'path' => 'requests' . DS . $this->Request->id . DS . 'source',
+                        ));
+                        
+                        // Add the file upload
+                        $errors = array();
+                        if (!$this->Request->addUpload($this->data, $errors)) {
+                                $this->Request->delete();
+                                $this->Session->setFlash(implode("<br />\n", $errors));
+                                $this->redirect(array('action'=>'add'));
+                        }
 
                         // Add the jobs to the request
+                        $jobs = array();
                         foreach ($this->data['Request']['App'] as $app) {
                                 list($platform_id, $doctype_code, $application_id, $version) = explode('_', $app);
-                                $this->data['Request']['Job'][] = compact('platform_id', 'doctype_code', 'application_id', 'version');
-                        }
-
-                        // Save the request
-                        $errors = array();
-                        if (!$this->Request->add($this->data, $errors)) {
-                                $this->Session->setFlash(implode("<br />\n", $errors));
+                                $jobs[] = compact('platform_id', 'application_id', 'version');
+                        }
+
+                        $this->Request->read();
+                        if ($this->Request->addJobs($jobs) == 0) {
+                                $this->Request->delete();
+                                $this->Session->setFlash(__('None of the jobs could be created. The request has been cancelled.', true));
                                 $this->redirect(array('action'=>'add'));
                         }
 
-                        if (sizeof($errors)) {
-                                $this->Session->setFlash(implode("<br />\n", $errors));
-                        }
-
+                        // Add ODF Validators
+                        $this->Request->addValidators();
+
+                        // Queue the preprocessor
+                        if (!$this->Request->defer('run', 'Preprocessor')) {
+                                $errors[] = __('Failed to queue the request for the virus scanner. Please contact system administration.', true);
+                        }
+
+                        // All done!
                         $this->redirect(array('action' => 'view', 'id' => $this->Request->id));
                 }
@@ -247 +277 @@
                 $doctypes = $this->Worker->Application->Doctype->find('all');
                 $formats = $this->Request->Format->find('list');
-                // $formatmap = $this->Request->Format->find('list', array('fields' => array('Format.id', 'Format.code')));
                 $mimetypes = $this->Mimetype->find('all');
 
@@ -273 +302 @@
 
                 // This is the tricky bit. Set the highest version for every application/platform combo as default
-                // This requires the workers array to be sorted my platform, app, version
+                // This requires the workers array to be sorted by platform, app, version
                 $short_id = false;
                 $platform_id = '';
@@ -310 +339 @@
                 $this->view = 'Media';
                 $this->set(array(
-                        'id' => $filename,
-                        'name' => substr($filename, 0, strrpos($filename, '.')),
+                        'id' => $id,
+                        'name' => $filename,
                         'download' => true,
                         'extension' => 'zip',

trunk/server/www/app/controllers/results_controller.php

@@ -120 +120 @@
 
                 $result = $this->_getResult($id);
-                $extpos = strrpos($result['Result']['filename'], '.');
-                $filename = substr($result['Result']['filename'], 0, $extpos);
-                $extension = substr($result['Result']['filename'], $extpos + 1);
-                $directory = $this->Result->Behaviors->File->settings['directory'];
+                $path = $this->Result->getPath();
+                $filename = basename($path);
+                $extpos = strrpos($filename, '.');
+                $name = substr($filename, 0, $extpos);
+                $extension = substr($filename, $extpos + 1);
 
                 $this->view = 'Media';
                 $this->set(array(
-                        'id' => $result['Result']['path'],
-                        'name' => $filename,
+                        'id' => $filename,
+                        'name' => $name,
                         'download' => true,
                         'extension' => $extension,
-                        'path' => $directory . DS,
+                        'path' => dirname($path) . DS,
                         // NOTE: This short-circuits CakePHP's crude filetype check
                         'mimeType' => array($extension => $result['Mimetype']['name']),

trunk/server/www/app/models/behaviors/file.php

@@ -66 +66 @@
 
                 if (!is_writeable($this->settings['directory'])) {
-                        $this->errors[] = __('File storage directory does not exist.', true);
-                        return;
-                }
-
-                if (!is_writeable($this->settings['directory'])) {
                         $this->errors[] = __('File storage directory is not writeable.', true);
                         return;
@@ -127 +122 @@
                 }
 
-                $destination = $this->_generatePath($file['name']);
-                if (!move_uploaded_file($file['tmp_name'], $this->settings['directory'] . DS . $destination)) {
+                $destination = $this->settings['directory'] . DS . $model->data[$model->name][$this->settings['fields']['path']] . DS . $file['name'];
+                if (!mkdir(dirname($destination), 0775, true)) {
+                        $this->errors[] = __('Could not create the destination directory.', true);
+                        return false;
+                }
+
+                if (!move_uploaded_file($file['tmp_name'], $destination)) {
                         $this->errors[] = __('The uploaded file cannot be moved.', true);
                         return false;
@@ -136 +136 @@
                         $this->settings['fields']['filename'] => $file['name'],
                         $this->settings['fields']['mimetype_id'] => $mimetype['Mimetype']['id'],
-                        $this->settings['fields']['path'] => $destination
                 ));
                 
                 return true;
-        }
-
-        /**
-         * Set the file records to any file
-         *
-         * @param object $model A reference to the model
-         * @param string $path full path to the file
-         * @param boolean $move True if the file should be moved, false if it should be copied.
-         * @return boolean True on success or false on failure
-         */
-        public function setFile(&$model, $path = false, $move = false)
-        {
-                if (!$path) {
-                        $this->errors[] = __('There was no file', true);
-                        return false;
-                }
-
-                if (!is_readable($path)) {
-                        $this->errors[] = __('The file cannot be read.', true);
-                        return false;
-                }
-
-                $mimestring = '';
-                if (!$mimetype = $this->_getMimetype($model, $path, $mimestring)) {
-                        $this->errors[] = sprintf(__('Invalid mimetype %s detected for file', true), $mimestring);
-                        return false;
-                }
-
-                if (!$this->_checkMimetype($mimetype)) {
-                        $this->errors[] = __('Illegal mimetype.', true);
-                        return false;
-                }
-
-                $filename = basename($path);
-                $destination = $this->_generatePath($filename);
-                if ($move) {
-                        $result = rename($path, $this->settings['directory'] . DS . $destination);
-                } else {
-                        $result = copy($path, $this->settings['directory'] . DS . $destination);
-                }
-
-                if (!$result) {
-                        $this->errors = __('The file could not be copied or moved.');
-                        return false;
-                }
-
-                $model->set(array(
-                        $this->settings['fields']['filename'] => $filename,
-                        $this->settings['fields']['mimetype_id'] => $mimetype['Mimetype']['id'],
-                        $this->settings['fields']['path'] => $destination
-                ));
-                
-                return True;
         }
 
@@ -205 +151 @@
         public function setFileBuffer(&$model, &$buffer, $filename, $filters = false)
         {
-                $destination_file = $this->_generatePath($filename);
-                $destination_path = $this->settings['directory'] . DS . $destination_file;
-
-                if (!$file = fopen($destination_path, 'w')) {
+                $destination = $this->settings['directory'] . DS . $model->data[$model->name][$this->settings['fields']['path']] . DS . $filename;
+
+                if (!file_exists(dirname($destination)) && !mkdir(dirname($destination), 0775, true)) {
+                        $this->errors[] = __('Could not create the destination directory.', true);
+                        return false;
+                }
+
+
+                if (!$file = fopen($destination, 'w')) {
                         $this->errors[] = __('Cannot write file from buffer.', true);
                         return false;
@@ -227 +178 @@
                 
                 $mimestring = '';
-                if (!$mimetype = $this->_getMimetype($model, $destination_path, $mimestring)) {
+                if (!$mimetype = $this->_getMimetype($model, $destination, $mimestring)) {
                         $this->errors[] = sprintf(__('Invalid mimetype %s detected for file buffer', true), $mimestring);
-                        unlink($destination_path);
+                        unlink($destination);
                         return false;
                 }
@@ -235 +186 @@
                 if (!$this->_checkMimetype($mimetype)) {
                         $this->errors[] = __('Illegal mimetype.', true);
-                        unlink($destination_path);
+                        unlink($destination);
                         return false;
                 }
@@ -242 +193 @@
                         $this->settings['fields']['filename'] => $filename,
                         $this->settings['fields']['mimetype_id'] => $mimetype['Mimetype']['id'],
-                        $this->settings['fields']['path'] => $destination_file
                 ));
                 
@@ -260 +210 @@
                 }
 
-                $path = $model->field('path');
-                if (!$path && !is_file($this->settings['directory'] . DS . $path)) {
+                if (($path = $this->getPath($model)) == false) {
                         return true; // There is no file
                 }
 
-                if (!is_writeable($this->settings['directory'] . DS . $path)) {
+                if (!is_writeable($path)) {
                         $this->errors[] = __('The file cannot be deleted.', true);
                         return false;
                 }
 
-                return unlink($this->settings['directory'] . DS . $path);
+                return unlink($path);
         }
 
@@ -285 +234 @@
                 }
                 
-                $path = $model->field('path');
-                if (!$path && !is_file($this->settings['directory'] . DS . $path)) {
-                        return null; // There is no file
-                }
-
-                return file_get_contents($this->settings['directory'] . DS . $path);
+                if ($path = $this->getPath($model)) {
+                        return file_get_contents($this->settings['directory'] . DS . $path);
+                }
+
+                return null;
         }
 
@@ -302 +250 @@
         {
                 if ($model->id) {
-                        $path = $model->field($this->settings['fields']['path']);
+                        // Load the data from the database
+                        $path     = $model->field($this->settings['fields']['path']);
+                        $filename = $model->field($this->settings['fields']['filename']);
                 } elseif (isset($model->data[$model->name][$this->settings['fields']['path']])) {
-                        $path = $model->data[$model->name][$this->settings['fields']['path']];
+                        // Get the data from the currently assigned model data
+                        $path     = $model->data[$model->name][$this->settings['fields']['path']];
+                        $filename = $model->data[$model->name][$this->settings['fields']['filename']];
                 } else {
-                        return false;
-                }
-                
-                if (!$path || !is_file($this->settings['directory'] . DS . $path)) {
+                        // Can't retrieve data
+                        return false;
+                }
+                
+                if (!$path || !$filename || !is_file($this->settings['directory'] . DS . $path . DS . $filename)) {
                         return false; // There is no file
                 }
 
-                return $this->settings['directory'] . DS . $path;
-        }
-
-        /**
-         * Generate a new full path to write to
-         */
-        protected function _generatePath($filename)
-        {
-                $path = String::uuid();
-
-                $this->log('_generatePath: uuid is ' . $path, LOG_DEBUG);
-                $this->log('_generatePath: filename is ' . $filename, LOG_DEBUG);
-                if ($filename) {
-                        $ext = strrchr($filename, '.');
-                        $this->log('_generatePath: ext is ' . $ext, LOG_DEBUG);
-                        if ($ext !== false) {
-                                $path .= $ext;
-                        }
-                }
-
-                return $path;
+                return $this->settings['directory'] . DS . $path . DS . $filename;
         }
 
@@ -382 +315 @@
         public function beforeSave(&$model)
         {
-                if (!isset($model->data[$model->name][$this->settings['fields']['path']])) {
+                if (!isset($model->data[$model->name][$this->settings['fields']['filename']])) {
                         return true; // No file is being saved
                 }
 
-                $filename = $model->data[$model->name][$this->settings['fields']['path']];
-
-                if ($filename && !file_exists($this->settings['directory'] . DS . $filename)) {
+                $path = $this->getPath($model);
+                if ($path && !file_exists($path)) {
                         $this->errors[] = __('The filename field is set but the file does not exist', true);
                 }
@@ -407 +339 @@
         public function beforeDelete(&$model)
         {
-                $filename = $this->settings['directory'] . DS . $model->field($this->settings['fields']['path']); 
-                if (file_exists($filename)) {
-                        if (!unlink($filename)) {
+                $path = $this->getPath($model);
+                if (file_exists($path)) {
+                        if (!unlink($path)) {
                                 $this->errors[] = __('The file could not be deleted.', true);
                         }

trunk/server/www/app/models/request.php

@@ -53 +53 @@
                 'Containable',
                 'BeanStalk.Deferrable',
-                'File' => 'files/requests',
+                'File' => 'files',
                 'Pipeline' => 'Preprocessor',
         );
@@ -75 +75 @@
 
         /**
-         * Create a new request with underlying jobs
+         * Add an upload to the request
          *
          * If the add fails, &$errors contains the error messages
@@ -83 +83 @@
          * @return boolean Success
          */
-        public function add($data, &$errors = array())
+        public function addUpload($data, $anonymise = false, &$errors = array())
         {
                 if (empty($data)) {
@@ -89 +89 @@
                         return false;
                 }
-                $this->set($data);
-                $this->set(array(
-                        'expire' => date('Y-m-d H:i:s', time() + Configure::read('Request.expire')),
-                        'state'  => self::STATE_PREPROCESSOR_QUEUED
-                ));
 
                 // Check that we have a file and jobs
-                if ((!isset($data['Request']['FileUpload']) || !is_array($data['Request']['FileUpload'])) && !isset($data['Request']['File']) && !isset($data['Request']['FileBuffer'])) {
+                if ((!isset($data['Request']['FileUpload']) || !is_array($data['Request']['FileUpload']))) {
                         $errors[] = __('You did not submit a file.', true);
                         return false;
                 }
 
-                if (!isset($data['Request']['Job']) || !is_array($data['Request']['Job']) || empty($data['Request']['Job'])) {
-                        $errors[] = __('You did not select any valid applications', true);
-                        return false;
-                }
-
-                // Add the file
-                if (isset($data['Request']['FileUpload'])) {
-                        if (!$this->setFileUpload($data['Request']['FileUpload'])) {
-                                $errors[] = __('The file upload failed. Please try again.', true);
-                                $errors = array_merge($errors, $this->Behaviors->File->errors);
-                                return false;
-                        }
-                } elseif (isset($data['Request']['File'])) {
-                        if (!$this->setFile($data['Request']['File'])) {
-                                $errors[] = __('Adding the file failed. Please try again.', true);
-                                $errors = array_merge($errors, $this->Behaviors->File->errors);
-                                return false;
-                        }
-                } else {
-                        if (!$this->setFileBuffer($data['Request']['FileBuffer']['contents'], $data['Request']['FileBuffer']['filename'])) {
-                                $errors[] = __('Adding the file buffer failed. Please try again.', true);
-                                $errors = array_merge($errors, $this->Behaviors->File->errors);
-                                return false;
-                        }
+                if (!$this->setFileUpload($data['Request']['FileUpload'])) {
+                        $errors[] = __('The file upload failed. Please try again.', true);
+                        $errors = array_merge($errors, $this->Behaviors->File->errors);
+                        return false;
                 }
 
                 // Run the ODF anonymiser if requested
-                if ($this->data['Request']['anonymise']) {
+                if ($data['Request']['anonymise']) {
                         if (!$this->anonymise()) {
+                                $this->deleteFile();
                                 $errors[] = __('The Anonymiser failed to run', true);
                                 return false;
                         }
                 }
-                unset($this->data['Request']['anonymise']);
-
-                // Save the request
-                if (($this->data = $this->save()) === false) {
-                        $this->deleteFile();
-                        $errors[] = __('The request could not be saved.', true);
-                        return false;
-                }
+                unset($data['Request']['anonymise']);
+
+                return $this->save();
+        }
+
+        /**
+         * Add jobs to this request
+         *
+         * @param array $jobs An array of jobs
+         * @return int The number of jobs created
+         */
+        public function addJobs($jobs = array())
+        {       
+                // Load the Application model. We need that below.
+                $Application = ClassRegistry::init('Application');
 
                 // Find the Mimetype associated with the request
@@ -149 +132 @@
                 )));
 
-                // Create jobs for this request
-                $jobErrors = 0;
-                foreach ($data['Request']['Job'] as $job) {
+                $jobCount = 0;
+                foreach ($jobs as $job) {
                         $job['request_id'] = $this->id;
 
-                        // If the job contains a doctype_code, check it
-                        if ($job['doctype_code']) {
-                                if ($job['doctype_code'] != $mimetype['Doctype']['code']) {
-                                        $jobErrors++;
-                                        continue;
-                                }
+                        // Check the request mimetype against the supported doctype
+                        $count = $Application->find('count', array(
+                                'contain' => array(
+                                        'Application.id' => $job['application_id'],
+                                        'Doctype.code' => $mimetype['Doctype']['code']
+                                ),
+                        ));
+
+                        if ($count == 0) {
+                                continue;
                         }
 
                         $this->Job->create();
-                        if (!$this->Job->save(array('Job' => $job))) {
-                                $jobErrors++;
-                        }
-                }
-
-                if ($jobErrors == sizeof($data['Request']['Job'])) {
-                        $this->delete();
-                        $errors[] = __('None of the jobs could be created. The request has been cancelled.', true);
-                        return false;
-                }
-
-                if ($jobErrors) {
-                        $errors[] = __('Not all jobs could be created.', true);
-                }
-
-                // Create validators for this request
+                        if ($this->Job->save(array('Job' => $job))) {
+                                $jobCount++;
+                        }
+                }
+
+                return $jobCount;
+        }
+
+        /**
+         * Add ODF validators for the request
+         */
+        public function addValidators()
+        {
                 $validators = Configure::read('Validator');
                 foreach ($validators as $validator_name => $validator_config) {
@@ -191 +174 @@
                         )));
                 }
-
-                // Schedule the preprocessor to run
-                if (!$this->defer('run', 'Preprocessor')) {
-                        $errors[] = __('Failed to queue the request for the virus scanner. Please contact system administration.', true);
-                }
-
-                return true;
         }
 
@@ -236 +212 @@
          * Generate a zipfile containing the original request and all results
          *
-         * @return array an array containing the directory and filename of the zipfile or null
+         * @return array an array containing the directory, id and filename of the zipfile to be used by the media view, or null
          */
         public function createZip()
@@ -244 +220 @@
                 }
 
-                $directory = $this->Behaviors->File->settings['directory'];
+                $root = $this->Behaviors->File->settings['directory'] . DS . $this->field('root');
                 $filename = $this->field('filename');
-                $zipdir = 'officeshots-' . substr($filename, 0, strrpos($filename, '.'));
-                $filename = $zipdir . '.zip';
-
-                if (file_exists($directory . DS . $filename)) {
-                        unlink($directory . DS . $filename);
-                }
-                $zip = new ZipArchive();
-                $zip->open($directory . DS . $filename, ZIPARCHIVE::CREATE);
-                $zip->addFromString($zipdir . '/' . $this->field('filename'), $this->getContents());
-
-                $jobs = $this->Job->find('all', array(
-                        'conditions' => array(
-                                'Job.request_id' => $this->id,
-                                'Job.result_id <>' => '',
-                        )
-                ));
-
-                foreach ($jobs as $job) {
-                        $appname = $job['Application']['name'] . '_'
-                                 . $job['Job']['version']
-                                 . '_' . $job['Platform']['name'];
-                        $appname = preg_replace('/[^a-z0-9._-]/', '_', low($appname));
-                        $appname = preg_replace('/_{2,}/', '_', $appname);
-
-                        $this->Job->Result->id = $job['Result']['id'];
-                        $extension = substr($job['Result']['filename'], strrpos($job['Result']['filename'], '.') + 1);
-                        $zip->addFromString($zipdir . '/' . $appname . '.' . $extension, $this->Job->Result->getContents());
-                }
-                $zip->close();
-
-                return compact('directory', 'filename');
+                $filename = substr($filename, 0, strrpos($filename, '.'));
+                $directory = TMP . 'zips';
+                $id = $this->id . '.zip';
+
+                if (file_exists($directory . DS . $id)) {
+                        unlink($directory . DS . $id);
+                }
+
+                $cwd = getcwd();
+                chdir($root);
+                shell_exec('cd "' . $root . '" && zip -r "' . $directory . DS . $id . '" .');
+                chdir($cwd);
+
+                return compact('id', 'directory', 'filename');
         }
 

trunk/server/www/app/models/result.php

@@ -55 +55 @@
                 'Containable',
                 'BeanStalk.Deferrable',
-                'File' => 'files/results',
+                'File' => 'files',
                 'Pipeline' => 'Postprocessor',
         );