KEELELIIN-40

config_dist.js

-/**
- * Created by priit on 26.05.15.
- */
 
 var config = {};
 
@@ -16,10 +13,18 @@ config.serviceTypes = {
 };
 
 config.availableWappers = {
-    CONTENT_TOKENIZER : 'contentTokenizer'
+    CONTENT_TOKENIZER : {
+        class: 'contentTokenizer',
+        commandTemplate: 'python /home/priit/Programs/KEELELIIN/pyutil/tokenizer.py -i [data] -o [outputPath1]'
+    },
+    MORFYHESTAJA : {
+        class: 'morfyhestaja',
+        commandTemplate: '/home/priit/Downloads/morfyhestaja/./parse.sh [data]'
+    }
 };
 
 config.paramUsageTypes = {
+    META: 'meta', //ei kasutata utiliidi parameetrina
     STRING : 'string', //parameeter  asendatakse väärtusega
     FILE : 'file' //parameeteri väärtus salvestatakse faili ja faili pathi kasutatakse argumendina
 };
@@ -28,30 +33,24 @@ config.service = {
     title: 'Wrapper',
     description: 'Wrapperi kirjeldus',
 
-    //Siia peab olema märgitud kõik võimalikud metaandmed ja parameetrid
-    serviceRequestTemplate: {
-        service: {
-            meta: { //key: value
-                isAsync: null // vaikimisi on teenus asünkroone,
-            },
-            params: {
-                //data: null,
-                //showLineNumbers: null
-            },
-            pipecontent: null
-        }
+    //Siia peab olema märgitud kõik võimalikud  parameetrid va sisendfailid
+    requestBodyTemplate: {
+        is_async: null//,
+        //someparam: null
     },
 
     //Iga parameeter peab ka siin esindatud olema
-    paramsMappings: {
-        /*data: {
-         usageType: config.paramUsageTypes.FILE,
-         filter: function(value){ return value; },
-         required: true,
-         allowEmpty: false,
-         validator: function(value, request){ return true; }
-         },*/
-        /*showLineNumbers: {
+    requestBodyParamsMappings: {
+        is_async: {
+            usageType: config.paramUsageTypes.META,
+            filter: function(value){
+                return value == 1;
+            },
+            required: true,
+            allowEmpty: false,
+            validator: function(value, request){ return true; }
+        }/*,
+         someparam: {
          usageType: config.paramUsageTypes.STRING,
          filter: function(value){
          if(value == 'yes'){
@@ -59,30 +58,25 @@ config.service = {
          }
          return null;
          },
-         required: true,
-         allowEmpty: false,
+         required: false,
+         allowEmpty: true,
          validator: function(value, request){ return true; }
          }*/
-
     },
 
-    pipecontentMapping: {
-        validator: function(value, request){
-            if(!value || !value.content){
-                request.setMessage('pipecontent', 'Puuduvad lähteandmed');
-            }
-            return true;
-        }
+    requestFiles: {
+        content: null
     },
 
-    staticOptions: {
-        wrapper: config.availableWappers.CONTENT_TOKENIZER,
-        type: config.serviceTypes.LOCAL, //Kohaliku commandline programm
-        commandTemplate: 'python /home/priit/Programs/KEELELIIN/pyutil/tokenizer.py -i [data] -o [outputPath]',
-        isAsync: null, //väärtustega true/false saab päringu väärtuse üle kirjutada
-        storagePath: "/home/priit/wrapper"
+    staticParams: {
+        uniqueId: 'wrapper_1',
+        wrapper: config.availableWappers.MORFYHESTAJA.class,
+        storagePath: "/home/priit/wrapper",
+        tmpPath: "/tmp/wrapper/",
+        sessionMaxLifetime: 600, //(s) Sessioonid, mille failide viimased muudatused on vanemad kui antud aeg, kustutatakse süsteemist
+        //siin saab päringu parameetrite väärtused üle kirjutada
+        isAsync: undefined //väärtustega true/false saab päringu väärtuse üle kirjutada
     }
-
 };
 
 module.exports = config;
\ No newline at end of file

controllers/api/v1/service.js

@@ -2,29 +2,30 @@ var logger = require('log4js').getLogger('service_controller');
 var express = require('express');
 var router = express.Router();
 var ServiceRequest = require(__base + '/src/model/serviceRequest');
-var WrapperService = require(__base + '/src/service/wrapperService');
+var wrapperService = require(__base + '/src/service/wrapperService');
+var fs = require('fs');
 
 router.post('/', function ( req, res ) {
 
-    var serviceRequest = new ServiceRequest( req.body );
+    logger.debug(req.body);
+    logger.debug(req.files);
+
+    var serviceRequest = new ServiceRequest( req.body, req.files );
 
     if(!serviceRequest.isValid()){
         res.send(serviceRequest.getMessages());
         return;
     }
 
-    var wrapperService = new WrapperService();
-    wrapperService.execute( serviceRequest.getData() , function (err, data) {
+    wrapperService.execute( serviceRequest, function (err, data) {
         if(err) return res.send({errors: err});
         res.send(data);
     });
 });
 
-router.get('/:instanceId', function(req, res) {
-
-    var wrapperService = new WrapperService();
-    wrapperService.getServiceResponse(req.params.instanceId, function (err, data) {
+router.get('/:sessionId', function(req, res) {
 
+    wrapperService.getServiceResponse(req.params.sessionId, function (err, data) {
         if(err){
             return res.status(404).send({errors: err});
         }
@@ -32,4 +33,28 @@ router.get('/:instanceId', function(req, res) {
     });
 });
 
+router.get('/:sessionId/:fileId', function(req, res) {
+
+    logger.debug({request:'Get file', instance: req.params.sessionId, file: req.params.fileId});
+
+    wrapperService.getServiceFile(req.params.sessionId, req.params.fileId, function (err, filePath) {
+        if(err){
+            return res.status(404).send({errors: err});
+        }
+
+        var stat = fs.statSync(filePath);
+        res.writeHead(200,{
+            'Content-Length': stat.size
+        });
+
+        var readStream = fs.createReadStream(filePath);
+        readStream.on('open', function () {
+            readStream.pipe(res);
+        });
+        readStream.on('error', function(err) {
+            res.end(err);
+        });
+    });
+});
+
 module.exports = router;

package.json

@@ -16,9 +16,11 @@
     "log4js": "^0.6.26",
     "morgan": "~1.5.3",
     "multer": "0.1.6",
+    "node-schedule": "^0.2.9",
     "randomstring": "^1.0.5",
     "redis": "^0.12.1",
-    "request": "^2.57.0"
+    "request": "^2.57.0",
+    "rimraf": "^2.4.0"
   },
   "devDependencies": {
     "assert": "^1.3.0",

src/mapper/commandModel.js

 var logger = require('log4js').getLogger('router_middleware');
-var config = require('../../config');
 var sessionService = require('../service/sessionService');
 
 function CommandModel(){
 
     var self = this;
     this.session = null;
+    this.serviceProperties = {};//näiteks local command template
+    this.outputPaths = {}; //väljundfailid
+    this.keyValues = {}; //teenuse parameetrid
 
     var fileValues = [];
-    this.outputPaths = {};
-    this.keyValues = {};
 
     this.init = function ( session ) {
         self.session = session;
@@ -19,13 +19,19 @@ function CommandModel(){
         self.keyValues[key] = value;
     };
 
+    /*
+    * Antud väärtus kirjutatakse faili
+    * */
     this.setFileValue = function (key, value) {
         fileValues.push(
             {key: key, value: value}
         );
     };
 
-    this.setOutputPath = function (key) {
+    /*
+    * Kasutatava programmi väljund
+    * */
+    this.addOutputPath = function (key) {
         self.outputPaths[key] = sessionService.getNewSessionFilePath( self.session );
     };
 
@@ -36,6 +42,8 @@ function CommandModel(){
         //store file values
         if(fileValues.length > 0){
             self._storeToFile(0, callback);
+        } else {
+            callback();
         }
     };
 

src/mapper/localCommand.js

 var logger = require('log4js').getLogger('local_command');
-var config = require('../../config');
 
 function LocalCommand( commandModel ) {
 
     var self = this;
 
-    this.commandModel = commandModel;
-    var commandTemplate = config.service.staticOptions.commandTemplate;
+    var keyValues = commandModel.keyValues;
+    var commandTemplate = commandModel.serviceProperties.commandTemplate;
+
     this.templateParams = commandTemplate.match(/\[(.*?)]/g);
-    this.commandParts = commandTemplate.split(' ');
+    var commandParts = commandTemplate.split(' ');
 
     this.command = "";
     this.commandParams = [];
 
-    this.generateLocalCommand = function () {
-
+    this.generate = function () {
         self._parseParams();
-
-        var paramsArray = this.commandParts.slice(this.commandParts);
+        var paramsArray = commandParts.slice(commandParts);
         self.command = paramsArray.shift();
-        self.commandParams = paramsArray.filter(function (value) { return value != ''; });
-        return true;
+        self.commandParams = paramsArray.filter(function (value) { return value != '' && value != null && value != undefined; });
+        return self;
     };
 
     this._parseParams = function () {
@@ -34,18 +32,17 @@ function LocalCommand( commandModel ) {
 
         var propertyItem = this.templateParams[index];
         var propertyKey = propertyItem.substr(1, (propertyItem.length - 2));
-        var value = self.commandModel.keyValues[propertyKey];
+        var value = keyValues[propertyKey];
         self._replacePropertyValue(propertyItem, value);
     };
 
     this._replacePropertyValue = function (propertyItem, value) {
 
-        for (i in this.commandParts) {
-            var part = this.commandParts[i];
-            self.commandParts[i] = part.replace(propertyItem, value);
+        for (i in commandParts) {
+            var part = commandParts[i];
+            commandParts[i] = part.replace(propertyItem, value);
         }
     };
-
 }
 
 module.exports = LocalCommand;
\ No newline at end of file

src/model/serviceRequest.js

 var logger = require('log4js').getLogger('service_request');
 var config = require('../../config');
 
-function ServiceRequest( requestData ) {
+function ServiceRequest( requestBody, requestFiles ) {
     var self = this;
 
-    this.data = requestData;
-    this.template = config.service.serviceRequestTemplate;
-    this.messages = null;
+    this.data = requestBody;
+    this.files = requestFiles;
+    var messages = null;
+
 
     this.isValid = function(){
-        self._mapMetaOptions();
         self._mapParams();
-        self._validatePipecontent();
-        return self.messages == null;
+        self._checkFiles();
+        return messages == null;
     };
 
     this.setMessage = function (key, value) {
 
-        if(self.messages == null){
-            self.messages = {}
-        }
-        self.messages[key] = value;
-    };
-
-    this._mapMetaOptions = function () {
-        var staticOptions = config.service.staticOptions;
-        if(staticOptions.isAsync === null ){
-            if(self.data.service.meta.isAsync === null || self.data.service.meta.isAsync === undefined){
-                self.data.service.meta.isAsync = self.template.service.meta.isAsync
-            }
-        } else {
-            self.data.service.meta.isAsync = staticOptions.isAsync;
+        if(messages == null){
+            messages = {}
         }
+        messages[key] = value;
     };
 
     this._mapParams = function(){
 
-        var expectedParams = this.template.service.params;
+        var staticParams = config.service.staticParams;
+
+        for(var property in config.service.requestBodyTemplate){
 
-        for(var property in expectedParams){
+            var value = self.data[property];
 
-            var value = self.data.service.params[property];
-            var mapping = config.service.paramsMappings[property];
+            var mapping = config.service.requestBodyParamsMappings[property];
+
+            if(staticParams[property] != undefined){
+                value = staticParams[property];
+            }
 
             if(mapping.filter){
                 value = mapping.filter(value);
             }
             if(mapping.required == true && value == undefined){
                 self.setMessage(property, 'Väli on nõutud');
+                continue;
             }
-            if(mapping.allowEmpty == false && (value == null || value == '') ){
+            if(mapping.allowEmpty == false && (value === null || value === '') ){
                 self.setMessage(property, 'Väli on täitmata');
+                continue;
             }
             if(mapping.validator){
                 mapping.validator(value, self);
             }
-            self.data.service.params[property] = value;
+            self.data[property] = value;
         }
     };
 
-    this._validatePipecontent = function () {
-        if(config.service.pipecontentMapping.validator){
-            config.service.pipecontentMapping.validator(self.data.service.pipecontent, self);
+    this._checkFiles = function () {
+
+        for( var fileId in config.service.requestFiles){
+            var file = this.files[fileId];
+            if(!file){
+                self.setMessage(fileId, 'Nõutud faili ei saadetud');
+            }
         }
     };
 
     this.getMessages = function(){
-        return {errors: self.messages, success: false};
-    };
-
-    this.getData = function(){
-        return self.data;
+        return {errors: messages, success: false};
     };
 }
 

src/model/session.js

 /**
  * Created by priit on 1.06.15.
  */
-function Session( id ){
+var Session = function( id ){
     this.isAsync = false;
     this.isFinished = false;
     this.id = id;
     this.success = false;
     this.message = Session.messages.NOT_FOUND;
     this.recheckInterval = 1;
-
+    this.requestBody = {};
+    this.requestFiles = {};
     this.data = null;
-    this.outputPath = null;
-}
+    this.outputFiles = {};
+};
+
+Session.prototype.addOutputFile = function(key, path){
+    this.outputFiles[key] = path;
+};
 
 Session.messages = {
     RUNNING: 'RUNNING',

src/service/cleanerService.js

+/**
+ * Created by priit on 29.06.15.
+ */
+var logger = require('log4js').getLogger('cleaner_service');
+var schedule = require('node-schedule');
+var config = require('./../../config');
+var sessionService = require('./sessionService');
+var fs = require('fs');
+
+var CleanerService = function () {
+
+    var self = this;
+
+    this.init = function(){
+        logger.debug('init');
+
+        if(!config.service.staticParams.sessionMaxLifetime){
+            return;
+        }
+
+        schedule.scheduleJob('0 0 * * *', function(){
+            self.cleanSystem();
+        });
+        self.cleanSystem();
+    };
+
+    this.cleanSystem = function () {
+        logger.debug('clean A');
+
+        var folder = config.service.staticParams.storagePath;
+
+        fs.readdir(folder, function(err, files) {
+            console.log('readFolder');
+
+            if(files.length > 0){
+                self.checkOnIndex(0, files, folder, function (err) {
+                    if(err){
+                        return logger.error(err);
+                    }
+                    return logger.info('Cleaned');
+                });
+            }
+        });
+    };
+
+    this.checkOnIndex = function(index, files,folder, cb){
+
+        if(index >= files.length){
+            return cb();
+        }
+
+        var file = files[index];
+
+        var continueScan = function () {
+            index = index +1;
+            self.checkOnIndex(index, files,folder, cb);
+        };
+
+        if (file[0] !== '.') {
+            var filePath = folder + '/' + file;
+            logger.debug(filePath);
+            fs.stat(filePath, function(err, stat) {
+                if (stat.isDirectory()) {
+                    var timeModified = stat.mtime.getTime();
+                    var currentTime = new Date().getTime();
+
+                    if(currentTime - timeModified > ( config.service.staticParams.sessionMaxLifetime * 1000 )){
+                        sessionService.removeSession(file, function (err) {
+                            if(err){
+                                logger.error('Session delete failed: ' + file);
+                            } else {
+                                logger.debug('Session delete success: ' + file);
+                            }
+                            continueScan();
+                        });
+                    } else {
+                        continueScan();
+                    }
+                } else {
+                    continueScan();
+                }
+            });
+        }
+    }
+};
+
+
+module.exports = new CleanerService();
\ No newline at end of file

src/service/daoService.js

 var logger = require('log4js').getLogger('dao_service');
 var redis = require('redis');
-//redis.debug_mode = true;
-
 var config = require(__base + 'config');
 
 var DaoService = function(){
     var self = this;
 
+    var prefix = config.service.staticParams.uniqueId + ':';
+
     this.client = redis.createClient(config.redis.port, config.redis.host, {});
 
     this.client.on('connect', function() {
@@ -19,44 +19,32 @@ var DaoService = function(){
 
     this.set =  function(key, value, cb){
 
-        this.client.hmset(key, value, function (err, reply) {
-            if(err){
-                logger.error(err);
-                return cb(err);
-            }
-
+        this.client.set(prefix + key, JSON.stringify(value), function (err, reply) {
             if(cb != undefined){
-                return cb();
+                cb(err, reply);
             }
         });
     };
 
     this.get = function(key, cb){
 
-        this.client.hgetall(key, function(err, reply) {
-            if(err){
-                logger.error(err);
-                return cb(err);
-            }
-            logger.debug('Got redis data');
-
+        this.client.get(prefix + key, function(err, reply) {
             if(cb != undefined){
-                return cb(null, reply);
+                if(err){
+                    return cb(err);
+                }
+                return cb(null, JSON.parse(reply));
             }
         });
     };
 
     this.delete = function(key, cb){
 
-        this.client.del(key, function(err, reply) {
-            if(err){
-                logger.error(err);
-            }
-
+        this.client.del(prefix + key, function (err, reply) {
             if(cb != undefined){
-                return cb();
+                cb(err, reply);
             }
-        });
+        } );
     };
 
     this.exists = function(key, cb){

src/service/executor/localExecutor.js

@@ -13,13 +13,11 @@ function LocalExecutor() {
 
         var response = {
             message: null,
-            success:true,
-            stdOutPath: sessionService.getNewSessionFilePath(commandModel.session),
-            outputPaths: commandModel.outputPaths
+            isSuccess:true,
+            stdOutPath: sessionService.getNewSessionFilePath(commandModel.session)