Changeset 297

Timestamp:

02/22/10 17:40:26 (5 months ago)

Author:

sander

Message:

Generate and display jobs and results for test suites

Location:

trunk/server/www/app

Files:

• config/core.php.default (modified) (3 diffs)
• config/sql/schema.php (modified) (2 diffs)
• controllers/galleries_controller.php (modified) (3 diffs)
• controllers/jobs_controller.php (modified) (3 diffs)
• controllers/validators_controller.php (modified) (5 diffs)
• models/gallery.php (modified) (1 diff)
• models/request.php (modified) (1 diff)
• models/testsuite.php (modified) (2 diffs)
• models/worker.php (modified) (3 diffs)
• vendors/shells/cron.php (modified) (2 diffs)
• views/elements/gallery.ctp (modified) (2 diffs)
• views/helpers/job_model.php (modified) (2 diffs)
• webroot/img/icons/image-format.png (added)
• webroot/img/icons/pdf-format.png (added)
• webroot/img/icons/roundtrip-format.png (added)

trunk/server/www/app/config/core.php.default

@@ -226 +226 @@
         Cache::config('default', array('engine' => 'File'));
 /**
+ * Set the default language
+ */
+        Configure::write('Config.language', 'eng');
+/**
  * The administrator E-mail address
  */
@@ -288 +292 @@
                 Configure::write('Factory.polltime', (6 * 60));
         }
+/**
+ * The number of seconds a factory must be running uninterrupted before it is considered "stable". Testsuite jobs will only be
+ * generated for stable factories. This way experimetal, development or unstable factories will not be overloaded with hundreds
+ * of jobs.
+ */
+        Configure::write('Factory.stabletime', (60 * 60 * 24));
 /**
  * BeanStalk Configuration
@@ -346 +356 @@
  */
         Configure::write('Anonymiser.path', '/usr/bin/iodf-greek.py');
+/**
+ * Testsuite uid/gid settings
+ */
+        Configure::write('Testsuite.user_id', '');
+        Configure::write('Testsuite.group_id', '');
 ?>

trunk/server/www/app/config/sql/schema.php

@@ -1 +1 @@
 <?php 
 /* SVN FILE: $Id$ */
-/* App schema generated on: 2010-02-10 15:02:05 : 1265810465*/
+/* App schema generated on: 2010-02-22 13:02:55 : 1266842515*/
 class AppSchema extends CakeSchema {
         var $name = 'App';
@@ -47 +47 @@
                         'id' => array('type' => 'string', 'null' => false, 'default' => NULL, 'length' => 36, 'key' => 'primary'),
                         'name' => array('type' => 'string', 'null' => false, 'default' => NULL, 'length' => 40),
+                        'icon' => array('type' => 'string', 'null' => false, 'default' => NULL),
                         'code' => array('type' => 'string', 'null' => false, 'default' => NULL, 'length' => 3),
                         'indexes' => array('PRIMARY' => array('column' => 'id', 'unique' => 1))

trunk/server/www/app/controllers/galleries_controller.php

@@ -28 +28 @@
         
         /** The controller helpers */
-        public $helpers = array('Html', 'Form', 'Javascript', 'RequestModel', 'ResultModel', 'ValidatorModel');
+        public $helpers = array('Html', 'Form', 'Javascript', 'RequestModel', 'JobModel', 'ValidatorModel');
 
         /** The models this controller uses */
@@ -141 +141 @@
                                 'Request',
                                 'Request.Validator' => array('order' => 'Validator.name ASC'),
-                                'Request.Job',
+                                'Request.Job' => array('order' => array(
+                                        'Application.name ASC',
+                                        'Job.version ASC',
+                                        'Platform.name ASC',
+                                        'Format.name ASC',
+                                )),
                                 'Request.Job.Application',
+                                'Request.Job.Format',
                                 'Request.Job.Platform',
                                 'Request.Job.Result',
@@ -172 +178 @@
                                         'contain' => array(
                                                 'Application',
+                                                'Format',
                                                 'Platform',
                                                 'Result',
                                                 'Result.Validator'
+                                        ),
+                                        'order' => array(
+                                                'Application.name ASC',
+                                                'Job.version ASC',
+                                                'Platform.name ASC',
+                                                'Format.name ASC',
                                         ),
                                 ));

trunk/server/www/app/controllers/jobs_controller.php

@@ -332 +332 @@
                                         AND `Job`.`format_id` IN  (" . implode(', ', $formats) . ")
                                         AND `Request`.`state` = " . Request::STATE_QUEUED . "
-                                        AND `Request`.`expire` > '" . date('Y-m-d H:i:s') . "'
+                                        AND (`Request`.`expire` > '" . date('Y-m-d H:i:s') . "' OR `Request`.`expire` IS NULL)
                                         AND `Mimetype`.`doctype_id` IN (" . implode(', ', $doctypes) . ")
                                         AND (`Request`.`own_factory` = 0
                                                 OR (`Request`.`own_factory` = 1 AND `Request`.`user_id` = '" . $this->AuthCert->user('id') . "')
-                                        )");
+                                        )
+                                ORDER BY `Request`.`own_factory` DESC,
+                                        `Request`.`priority` ASC,
+                                        `Request`.`created` ASC
+                                LIMIT 0,1");
                         $jobs = array_merge($jobs, $jobSet);
                 }
@@ -344 +348 @@
                 }
 
-                $jobs = Set::sort($jobs, '{n}.Request.created', SORT_ASC);
-                $jobs = Set::sort($jobs, '{n}.Request.priority', SORT_ASC);
-                $jobs = Set::sort($jobs, '{n}.Request.own_factory', SORT_DESC);
+                usort($jobs, array($this, '_cmpJobs'));
                 $job = array_shift($jobs);
 
@@ -377 +379 @@
 
         /**
+         * A helper function to sort jobs for xmlrpc_poll()
+         */
+        public function _cmpJobs($a, $b)
+        {
+                // First, look at the own_factory flag
+                if ($a['Request']['own_factory'] == 1 && $b['Request']['own_factory'] == 0) {
+                        return -1;
+                }
+                if ($a['Request']['own_factory'] == 0 && $b['Request']['own_factory'] == 1) {
+                        return 1;
+                }
+
+                // Next, look at the priority
+                if ($a['Request']['priority'] < $b['Request']['priority']) {
+                        return -1;
+                }
+                if ($a['Request']['priority'] > $b['Request']['priority']) {
+                        return 1;
+                }
+
+                // If the priorities are also equal, look at the creation date
+                return ($a['Request']['created'] < $b['Request']['created']) ? -1 : 1;
+        }
+
+        /**
          * Finish a job by uploading the result
          *

trunk/server/www/app/controllers/validators_controller.php

@@ -37 +37 @@
         {
                 parent::beforeFilter();
-                if (Configure::read('Auth.allowAnonymous')) {
-                        $this->AuthCert->allow('view');
-                }
+                $this->AuthCert->allow('view');
         }
 
@@ -59 +57 @@
                 $this->Validator->contain(array(
                         'Request',
-                        'Request.Gallery',
                         'Result',
                         'Result.Job',
@@ -65 +62 @@
                         'Result.Job.Platform',
                         'Result.Job.Request',
-                        'Result.Job.Request.Gallery',
                 ));
                 
@@ -73 +69 @@
                 }
 
-                if ($validator['Request']['user_id'] == $this->AuthCert->user('id')) {
-                        return $validator;
-                }
-
-                if (!empty($validator['Request']['Gallery'])) {
+                if ($this->Validator->Request->checkAccess($this->AuthCert->user('id'), 'read', $validator['Request']['id'])) {
                         return $validator;
                 }
@@ -103 +95 @@
         {
                 $validator = $this->_getValidator($id);
+
+                // Strip HTML skeleton
+                if (preg_match('#^<html>.*<body>(.*)</body>#si', $validator['Validator']['response'], $match)) {
+                        $validator['Validator']['response'] = $match[1];
+                }
+
                 $this->set(array(
                         'validator' => $validator,

trunk/server/www/app/models/gallery.php

@@ -170 +170 @@
 
         /**
+         * Return all requests and jobs associated with a gallery and all subgalleries
+         * @param int $id The Gallery ID
+         * @return array
+         */
+        public function getRequests($id = null)
+        {
+                if (!$id) {
+                        $id = $this->id;
+                }
+
+                if (!$id) {
+                        return array();
+                }
+
+                $gallery = $this->find('first', array(
+                        'conditions' => array('Gallery.id' => $id),
+                        'recursive' => -1,
+                ));
+
+                $result = $this->GalleriesRequest->find('all', array(
+                        'conditions' => array('GalleriesRequest.gallery_id' => $id),
+                        'recursive' => -1,
+                ));
+                $request_ids = Set::extract('/GalleriesRequest/request_id', $result);
+
+                $result = $this->query('SELECT DISTINCT `GalleriesRequest`.`request_id`
+                        FROM `galleries_requests` AS `GalleriesRequest`
+                        LEFT JOIN `galleries` AS `Gallery`
+                                ON `GalleriesRequest`.`gallery_id` = `Gallery`.`id`
+                        WHERE
+                                `Gallery`.`lft` > ' . $gallery['Gallery']['lft'] . '
+                                AND `Gallery`.`rght` < ' . $gallery['Gallery']['rght']);
+                $subgallery_request_ids = Set::extract('/GalleriesRequest/request_id', $result);
+
+                $request_ids = Set::merge($request_ids, $subgallery_request_ids);
+
+                $requests = $this->Request->find('all', array(
+                        'contain' => array('Job', 'Mimetype'),
+                        'conditions' => array('Request.id' => $request_ids),
+                ));
+
+                return $requests;
+        }
+
+        /**
          * Convert the Markdown description to HTML before saving
          * @return boolean True to continue saving

trunk/server/www/app/models/request.php

@@ -164 +164 @@
          *
          * @param array $jobs An array of jobs
+         * @param string $id The request ID
          * @return int The number of jobs created
          */
-        public function addJobs($jobs = array())
-        {       
+        public function addJobs($jobs = array(), $id = null)
+        {
                 // Load the Application model. We need that below.
                 $Application = ClassRegistry::init('Application');
+
+                // Load the request
+                if (!$id) {
+                        $id = $this->id;
+                }
+
+                if (!$id) {
+                        return 0;
+                }
+
+                $request = $this->find('first', array(
+                        'conditions' => array('Request.id' => $id),
+                        'recursive' => -1,
+                ));
 
                 // Find the Mimetype associated with the request
                 // We can use this to check the doctype on the submitted jobs below
                 $mimetype = $this->Mimetype->find('first', array('conditions' => array(
-                        'Mimetype.id' => $this->data['Request']['mimetype_id'],
+                        'Mimetype.id' => $request['Request']['mimetype_id'],
                 )));
 
                 $jobCount = 0;
                 foreach ($jobs as $job) {
-                        $job['request_id'] = $this->id;
+                        $job['request_id'] = $id;
 
                         // Check the request mimetype against the supported doctype

trunk/server/www/app/models/testsuite.php

@@ -28 +28 @@
 
         /**
-         * Synchronise all testsuites to the database
+         * Synchronise all testsuite files to the database
          */
         public function synchronise()
@@ -247 +247 @@
                 }
         }
+
+        /**
+         * Add jobs to all the test suite requests for all stable factories
+         */
+        public function addJobs()
+        {
+                // Load the necessary models
+                $this->Worker = ClassRegistry::init('Worker');
+                $this->Request = ClassRegistry::init('Request');
+
+                $workers = $this->Worker->getActive(true);
+                $requests = $this->_getRequests();
+
+                foreach ($requests as $request) {
+                        $this->_addJobs($request, $workers);
+                }
+        }
+
+        /**
+         * Get all requests and jobs that belong to all test suites
+         */
+        private function _getRequests()
+        {
+                $requests = array();
+                $suites = $this->find('all', array('recursive' => -1));
+
+                foreach ($suites as $suite) {
+                        $requests = Set::merge(
+                                $requests,
+                                $this->Gallery->getRequests($suite['Testsuite']['gallery_id'])
+                        );
+                }
+
+                return $requests;
+        }
+
+        /**
+         * Add jobs for $workers to a single request
+         * @param array $workers An array of available workers
+         * @param array $request The request to add the jobs to
+         */
+        private function _addJobs($request, $workers)
+        {
+                $newJobs = array();
+                foreach ($workers as $worker) {
+                        // If the worker does not support this doctype, skip it.
+                        if ($request['Mimetype']['doctype_id'] != $worker['Doctype']['id']) {
+                                continue;
+                        }
+
+                        foreach ($worker['Format'] as $format) {
+                                $match = false;
+
+                                foreach ($request['Job'] as $job) {
+                                        $match = (
+                                                $job['platform_id'] == $worker['Platform']['id']
+                                                && $job['application_id'] == $worker['Application']['id']
+                                                && $job['version'] == $worker['Worker']['version']
+                                                && $job['format_id'] == $format['id']
+                                        );
+
+                                        // If a job already exists we're done checking
+                                        if ($match) {
+                                                break;
+                                        }
+                                }
+
+                                if ($match) {
+                                        continue;
+                                }
+
+                                // If we get here, no suitable job exists and we need to add one
+                                $newJobs[] = array(
+                                        'platform_id' => $worker['Platform']['id'],
+                                        'application_id' => $worker['Application']['id'],
+                                        'version' => $worker['Worker']['version'],
+                                        'format_id' => $format['id'],
+                                );
+                        }
+                }
+                
+                $this->Request->id = $request['Request']['id'];
+                $this->Request->addJobs($newJobs);
+        }
 }
 

trunk/server/www/app/models/worker.php

@@ -55 +55 @@
         /**
          * Get a list of all active worker types. That is, unique combinations of Platform, Doctype and Application
-         *
-         * @param integer $time Maximum time since the last poll, in minutes
+         * @param boolean $stable If true, return only stable factories
          * @return array
          */
-        public function getActive()
+        public function getActive($stable = false)
         {
                 App::import('Sanitize');
+                $time = date('Y-m-d H:i:s', time() - Configure::read('Factory.polltime'));
 
-                $time = date('Y-m-d H:i:s', time() - Configure::read('Factory.polltime'));
+                if ($stable) {
+                        $stableTime = date('Y-m-d H:i:s', time() - Configure::read('Factory.stabletime'));
+                        $sqlWhere = "AND `Factory`.`active_since` != '0000-00-00 00:00:00'
+                                     AND `Factory`.`active_since` < '$stableTime'
+                                     AND `Worker`.`development` = 0";
+                } else {
+                        $sqlWhere = '';
+                }
+
                 $active = $this->query("SELECT DISTINCT
                                 `Worker`.`version`,
@@ -82 +90 @@
                         LEFT JOIN `doctypes` AS `Doctype` ON (`ApplicationsDoctype`.`doctype_id` = `Doctype`.`id`)
                         WHERE `Factory`.`last_poll` > '$time'
+                                $sqlWhere
                         ORDER BY `Platform`.`name` ASC, `Application`.`name` ASC, `Worker`.`version` ASC");
 
@@ -100 +109 @@
                                         LEFT JOIN `applications_doctypes` AS `ApplicationsDoctype` ON `Application`.`id` = `ApplicationsDoctype`.`application_id`
                                         WHERE `Factory`.`last_poll` > '$time'
+                                                $sqlWhere
                                                 AND `Worker`.`version` = '" . Sanitize::escape($app['Worker']['version']) . "'
                                                 AND `Application`.`id` = '" . $app['Application']['id'] . "'

trunk/server/www/app/vendors/shells/cron.php

@@ -47 +47 @@
 
         /**
+         * Generate new jobs for the test suites. This should be run once per day.
+         */
+        public function updateTestsuites()
+        {
+                $Testsuite = ClassRegistry::init('Testsuite');
+                $this->out($Testsuite->addJobs());
+        }
+
+        /**
          * Print shell help
          */
@@ -57 +66 @@
                 $this->out('Commands:');
                 $this->out("\n\texpire\n\t\tSet STATE_EXPIRED on all requests whose expiry time has passed.\n\t\tThis should be run every minute.");
-                $this->out("\n\tsyncTestsuites\n\t\tSynchronise all ODF testsuites.\n\t\tThis should be run every hour.");
+                $this->out("\n\tupdateTestsuites\n\t\tCreate jopbs for all ODF testsuites.\n\t\tThis should be run every hour.");
                 $this->out("\n\thelp\n\t\tShow this help");
                 $this->out('');

trunk/server/www/app/views/elements/gallery.ctp

@@ -24 +24 @@
         </tr>
         <?php foreach ($request['Job'] as $job):?>
-        <tr class="result<?php if (!$job['Result']) { echo ' expired'; } ?>">
+        <tr class="result<?php if (!$job['Result'] || !$job['Result']['id']) { echo ' expired'; } ?>">
                         <td class="job-name" style="padding-left: <?php echo $indent + 2;?>em">
                                 <?php
-                                        if ($job['Result']) {
+                                        echo $jobModel->getFormatIcon($job);
+                                        if ($job['Result'] && $job['Result']['id']) {
                                                 echo $html->link($job['Application']['name'] . ' ' . $job['version'] . ' (' . $job['Platform']['name'] . ')', array('controller'=> 'results', 'action'=>'view', $job['Result']['id']));
                                         } else {
@@ -44 +45 @@
                         </td>
                         <td><?php echo isset($job['Result']['created']) ? $job['Result']['created'] : $job['created']; ?></td>
-                        <td><?php echo $resultModel->getState($job['Result']); ?></td>
+                        <td><?php echo $jobModel->getState($job, $request['state']); ?></td>
                         <?php if ($access):?>
                                 <td class="actions">&nbsp;</td>

trunk/server/www/app/views/helpers/job_model.php

@@ -36 +36 @@
                 }
 
-                if (!empty($job['Job']['Result'])) {
+                if (!empty($job['Job']['Result']) && !empty($job['Job']['Result']['id'])) {
                         return $this->output($job['Job']['Result']['Format']['name']);
+                }
+
+                return $this->getState($job, $request_state);
+        }
+
+        /**
+         * Return the state of a job
+         */
+        public function getState($job, $request_state)
+        {
+                if (!isset($job['Job'])) {
+                        $job = array('Job' => $job);
+                }
+
+                if (!empty($job['Job']['Result']) && !empty($job['Job']['Result']['id'])) {
+                        return $this->output(__('Finished', true));
                 }
 
@@ -87 +103 @@
                 return $this->output($this->Html->image('icons/error.png', array('alt' => __('Failed', true))));
         }
+
+        /**
+         * Return the small format icon of a job
+         */
+        public function getFormatIcon($job)
+        {
+                if (!isset($job['Job'])) {
+                        $job = array('Job' => $job);
+                }
+
+                if (empty($job['Job']['Format'])) {
+                        return;
+                }
+
+                return $this->output($this->Html->image('icons/' . $job['Job']['Format']['icon'], array(
+                        'alt' => $job['Job']['Format']['name'],
+                )) . ' ');
+        }
 }