Commit 99ef0687 authored by priit's avatar priit
Browse files

KEELELIIN-40

parent 6ca6c4fd
/**
* Created by priit on 26.05.15.
*/
var config = {};
......@@ -16,10 +13,18 @@ config.serviceTypes = {
};
config.availableWappers = {
CONTENT_TOKENIZER : 'contentTokenizer'
CONTENT_TOKENIZER : {
class: 'contentTokenizer',
commandTemplate: 'python /home/priit/Programs/KEELELIIN/pyutil/tokenizer.py -i [data] -o [outputPath1]'
},
MORFYHESTAJA : {
class: 'morfyhestaja',
commandTemplate: '/home/priit/Downloads/morfyhestaja/./parse.sh [data]'
}
};
config.paramUsageTypes = {
META: 'meta', //ei kasutata utiliidi parameetrina
STRING : 'string', //parameeter asendatakse väärtusega
FILE : 'file' //parameeteri väärtus salvestatakse faili ja faili pathi kasutatakse argumendina
};
......@@ -28,30 +33,24 @@ config.service = {
title: 'Wrapper',
description: 'Wrapperi kirjeldus',
//Siia peab olema märgitud kõik võimalikud metaandmed ja parameetrid
serviceRequestTemplate: {
service: {
meta: { //key: value
isAsync: null // vaikimisi on teenus asünkroone,
},
params: {
//data: null,
//showLineNumbers: null
},
pipecontent: null
}
//Siia peab olema märgitud kõik võimalikud parameetrid va sisendfailid
requestBodyTemplate: {
is_async: null//,
//someparam: null
},
//Iga parameeter peab ka siin esindatud olema
paramsMappings: {
/*data: {
usageType: config.paramUsageTypes.FILE,
filter: function(value){ return value; },
required: true,
allowEmpty: false,
validator: function(value, request){ return true; }
},*/
/*showLineNumbers: {
requestBodyParamsMappings: {
is_async: {
usageType: config.paramUsageTypes.META,
filter: function(value){
return value == 1;
},
required: true,
allowEmpty: false,
validator: function(value, request){ return true; }
}/*,
someparam: {
usageType: config.paramUsageTypes.STRING,
filter: function(value){
if(value == 'yes'){
......@@ -59,30 +58,25 @@ config.service = {
}
return null;
},
required: true,
allowEmpty: false,
required: false,
allowEmpty: true,
validator: function(value, request){ return true; }
}*/
},
pipecontentMapping: {
validator: function(value, request){
if(!value || !value.content){
request.setMessage('pipecontent', 'Puuduvad lähteandmed');
}
return true;
}
requestFiles: {
content: null
},
staticOptions: {
wrapper: config.availableWappers.CONTENT_TOKENIZER,
type: config.serviceTypes.LOCAL, //Kohaliku commandline programm
commandTemplate: 'python /home/priit/Programs/KEELELIIN/pyutil/tokenizer.py -i [data] -o [outputPath]',
isAsync: null, //väärtustega true/false saab päringu väärtuse üle kirjutada
storagePath: "/home/priit/wrapper"
staticParams: {
uniqueId: 'wrapper_1',
wrapper: config.availableWappers.MORFYHESTAJA.class,
storagePath: "/home/priit/wrapper",
tmpPath: "/tmp/wrapper/",
sessionMaxLifetime: 600, //(s) Sessioonid, mille failide viimased muudatused on vanemad kui antud aeg, kustutatakse süsteemist
//siin saab päringu parameetrite väärtused üle kirjutada
isAsync: undefined //väärtustega true/false saab päringu väärtuse üle kirjutada
}
};
module.exports = config;
\ No newline at end of file
......@@ -2,29 +2,30 @@ var logger = require('log4js').getLogger('service_controller');
var express = require('express');
var router = express.Router();
var ServiceRequest = require(__base + '/src/model/serviceRequest');
var WrapperService = require(__base + '/src/service/wrapperService');
var wrapperService = require(__base + '/src/service/wrapperService');
var fs = require('fs');
router.post('/', function ( req, res ) {
var serviceRequest = new ServiceRequest( req.body );
logger.debug(req.body);
logger.debug(req.files);
var serviceRequest = new ServiceRequest( req.body, req.files );
if(!serviceRequest.isValid()){
res.send(serviceRequest.getMessages());
return;
}
var wrapperService = new WrapperService();
wrapperService.execute( serviceRequest.getData() , function (err, data) {
wrapperService.execute( serviceRequest, function (err, data) {
if(err) return res.send({errors: err});
res.send(data);
});
});
router.get('/:instanceId', function(req, res) {
var wrapperService = new WrapperService();
wrapperService.getServiceResponse(req.params.instanceId, function (err, data) {
router.get('/:sessionId', function(req, res) {
wrapperService.getServiceResponse(req.params.sessionId, function (err, data) {
if(err){
return res.status(404).send({errors: err});
}
......@@ -32,4 +33,28 @@ router.get('/:instanceId', function(req, res) {
});
});
router.get('/:sessionId/:fileId', function(req, res) {
logger.debug({request:'Get file', instance: req.params.sessionId, file: req.params.fileId});
wrapperService.getServiceFile(req.params.sessionId, req.params.fileId, function (err, filePath) {
if(err){
return res.status(404).send({errors: err});
}
var stat = fs.statSync(filePath);
res.writeHead(200,{
'Content-Length': stat.size
});
var readStream = fs.createReadStream(filePath);
readStream.on('open', function () {
readStream.pipe(res);
});
readStream.on('error', function(err) {
res.end(err);
});
});
});
module.exports = router;
......@@ -16,9 +16,11 @@
"log4js": "^0.6.26",
"morgan": "~1.5.3",
"multer": "0.1.6",
"node-schedule": "^0.2.9",
"randomstring": "^1.0.5",
"redis": "^0.12.1",
"request": "^2.57.0"
"request": "^2.57.0",
"rimraf": "^2.4.0"
},
"devDependencies": {
"assert": "^1.3.0",
......
var logger = require('log4js').getLogger('router_middleware');
var config = require('../../config');
var sessionService = require('../service/sessionService');
function CommandModel(){
var self = this;
this.session = null;
this.serviceProperties = {};//näiteks local command template
this.outputPaths = {}; //väljundfailid
this.keyValues = {}; //teenuse parameetrid
var fileValues = [];
this.outputPaths = {};
this.keyValues = {};
this.init = function ( session ) {
self.session = session;
......@@ -19,13 +19,19 @@ function CommandModel(){
self.keyValues[key] = value;
};
/*
* Antud väärtus kirjutatakse faili
* */
this.setFileValue = function (key, value) {
fileValues.push(
{key: key, value: value}
);
};
this.setOutputPath = function (key) {
/*
* Kasutatava programmi väljund
* */
this.addOutputPath = function (key) {
self.outputPaths[key] = sessionService.getNewSessionFilePath( self.session );
};
......@@ -36,6 +42,8 @@ function CommandModel(){
//store file values
if(fileValues.length > 0){
self._storeToFile(0, callback);
} else {
callback();
}
};
......
var logger = require('log4js').getLogger('local_command');
var config = require('../../config');
function LocalCommand( commandModel ) {
var self = this;
this.commandModel = commandModel;
var commandTemplate = config.service.staticOptions.commandTemplate;
var keyValues = commandModel.keyValues;
var commandTemplate = commandModel.serviceProperties.commandTemplate;
this.templateParams = commandTemplate.match(/\[(.*?)]/g);
this.commandParts = commandTemplate.split(' ');
var commandParts = commandTemplate.split(' ');
this.command = "";
this.commandParams = [];
this.generateLocalCommand = function () {
this.generate = function () {
self._parseParams();
var paramsArray = this.commandParts.slice(this.commandParts);
var paramsArray = commandParts.slice(commandParts);
self.command = paramsArray.shift();
self.commandParams = paramsArray.filter(function (value) { return value != ''; });
return true;
self.commandParams = paramsArray.filter(function (value) { return value != '' && value != null && value != undefined; });
return self;
};
this._parseParams = function () {
......@@ -34,18 +32,17 @@ function LocalCommand( commandModel ) {
var propertyItem = this.templateParams[index];
var propertyKey = propertyItem.substr(1, (propertyItem.length - 2));
var value = self.commandModel.keyValues[propertyKey];
var value = keyValues[propertyKey];
self._replacePropertyValue(propertyItem, value);
};
this._replacePropertyValue = function (propertyItem, value) {
for (i in this.commandParts) {
var part = this.commandParts[i];
self.commandParts[i] = part.replace(propertyItem, value);
for (i in commandParts) {
var part = commandParts[i];
commandParts[i] = part.replace(propertyItem, value);
}
};
}
module.exports = LocalCommand;
\ No newline at end of file
var logger = require('log4js').getLogger('service_request');
var config = require('../../config');
function ServiceRequest( requestData ) {
function ServiceRequest( requestBody, requestFiles ) {
var self = this;
this.data = requestData;
this.template = config.service.serviceRequestTemplate;
this.messages = null;
this.data = requestBody;
this.files = requestFiles;
var messages = null;
this.isValid = function(){
self._mapMetaOptions();
self._mapParams();
self._validatePipecontent();
return self.messages == null;
self._checkFiles();
return messages == null;
};
this.setMessage = function (key, value) {
if(self.messages == null){
self.messages = {}
}
self.messages[key] = value;
};
this._mapMetaOptions = function () {
var staticOptions = config.service.staticOptions;
if(staticOptions.isAsync === null ){
if(self.data.service.meta.isAsync === null || self.data.service.meta.isAsync === undefined){
self.data.service.meta.isAsync = self.template.service.meta.isAsync
}
} else {
self.data.service.meta.isAsync = staticOptions.isAsync;
if(messages == null){
messages = {}
}
messages[key] = value;
};
this._mapParams = function(){
var expectedParams = this.template.service.params;
var staticParams = config.service.staticParams;
for(var property in config.service.requestBodyTemplate){
for(var property in expectedParams){
var value = self.data[property];
var value = self.data.service.params[property];
var mapping = config.service.paramsMappings[property];
var mapping = config.service.requestBodyParamsMappings[property];
if(staticParams[property] != undefined){
value = staticParams[property];
}
if(mapping.filter){
value = mapping.filter(value);
}
if(mapping.required == true && value == undefined){
self.setMessage(property, 'Väli on nõutud');
continue;
}
if(mapping.allowEmpty == false && (value == null || value == '') ){
if(mapping.allowEmpty == false && (value === null || value === '') ){
self.setMessage(property, 'Väli on täitmata');
continue;
}
if(mapping.validator){
mapping.validator(value, self);
}
self.data.service.params[property] = value;
self.data[property] = value;
}
};
this._validatePipecontent = function () {
if(config.service.pipecontentMapping.validator){
config.service.pipecontentMapping.validator(self.data.service.pipecontent, self);
this._checkFiles = function () {
for( var fileId in config.service.requestFiles){
var file = this.files[fileId];
if(!file){
self.setMessage(fileId, 'Nõutud faili ei saadetud');
}
}
};
this.getMessages = function(){
return {errors: self.messages, success: false};
};
this.getData = function(){
return self.data;
return {errors: messages, success: false};
};
}
......
/**
* Created by priit on 1.06.15.
*/
function Session( id ){
var Session = function( id ){
this.isAsync = false;
this.isFinished = false;
this.id = id;
this.success = false;
this.message = Session.messages.NOT_FOUND;
this.recheckInterval = 1;
this.requestBody = {};
this.requestFiles = {};
this.data = null;
this.outputPath = null;
}
this.outputFiles = {};
};
Session.prototype.addOutputFile = function(key, path){
this.outputFiles[key] = path;
};
Session.messages = {
RUNNING: 'RUNNING',
......
/**
* Created by priit on 29.06.15.
*/
var logger = require('log4js').getLogger('cleaner_service');
var schedule = require('node-schedule');
var config = require('./../../config');
var sessionService = require('./sessionService');
var fs = require('fs');
var CleanerService = function () {
var self = this;
this.init = function(){
logger.debug('init');
if(!config.service.staticParams.sessionMaxLifetime){
return;
}
schedule.scheduleJob('0 0 * * *', function(){
self.cleanSystem();
});
self.cleanSystem();
};
this.cleanSystem = function () {
logger.debug('clean A');
var folder = config.service.staticParams.storagePath;
fs.readdir(folder, function(err, files) {
console.log('readFolder');
if(files.length > 0){
self.checkOnIndex(0, files, folder, function (err) {
if(err){
return logger.error(err);
}
return logger.info('Cleaned');
});
}
});
};
this.checkOnIndex = function(index, files,folder, cb){
if(index >= files.length){
return cb();
}
var file = files[index];
var continueScan = function () {
index = index +1;
self.checkOnIndex(index, files,folder, cb);
};
if (file[0] !== '.') {
var filePath = folder + '/' + file;
logger.debug(filePath);
fs.stat(filePath, function(err, stat) {
if (stat.isDirectory()) {
var timeModified = stat.mtime.getTime();
var currentTime = new Date().getTime();
if(currentTime - timeModified > ( config.service.staticParams.sessionMaxLifetime * 1000 )){
sessionService.removeSession(file, function (err) {
if(err){
logger.error('Session delete failed: ' + file);
} else {
logger.debug('Session delete success: ' + file);
}
continueScan();
});
} else {
continueScan();
}
} else {
continueScan();
}
});
}
}
};
module.exports = new CleanerService();
\ No newline at end of file
var logger = require('log4js').getLogger('dao_service');
var redis = require('redis');
//redis.debug_mode = true;
var config = require(__base + 'config');
var DaoService = function(){
var self = this;
var prefix = config.service.staticParams.uniqueId + ':';
this.client = redis.createClient(config.redis.port, config.redis.host, {});
this.client.on('connect', function() {
......@@ -19,44 +19,32 @@ var DaoService = function(){
this.set = function(key, value, cb){
this.client.hmset(key, value, function (err, reply) {
if(err){
logger.error(err);
return cb(err);
}
this.client.set(prefix + key, JSON.stringify(value), function (err, reply) {
if(cb != undefined){
return cb();
cb(err, reply);
}
});
};
this.get = function(key, cb){
this.client.hgetall(key, function(err, reply) {
if(err){
logger.error(err);
return cb(err);
}
logger.debug('Got redis data');
this.client.get(prefix + key, function(err, reply) {
if(cb != undefined){
return cb(null, reply);
if(err){
return cb(err);
}
return cb(null, JSON.parse(reply));
}
});
};
this.delete = function(key, cb){
this.client.del(key, function(err, reply) {
if(err){
logger.error(err);
}
this.client.del(prefix + key, function (err, reply) {
if(cb != undefined){
return cb();
cb(err, reply);
}
});
} );
};
this.exists = function(key, cb){
......
......@@ -13,13 +13,11 @@ function LocalExecutor() {
var response = {
message: null,
success:true,
stdOutPath: sessionService.getNewSessionFilePath(commandModel.session),
outputPaths: commandModel.outputPaths
isSuccess:true,
stdOutPath: sessionService.getNewSessionFilePath(commandModel.session)