Commit b60e7d26 authored by priit's avatar priit
Browse files

Uued wrapperid + confi refactor

parent 7490ec5a
/**
* Kopeeri antud fail faili config.js
*
* require soovitud wrapperi conf ja kirjuta üle vajalikud parameetrid
*/
// var config = require('./wrapper_configs/config_tokenizer');
var config = require('./wrapper_configs/global');
/*
config.port = 8001;
config.log4js.appenders.push({
"type": "logLevelFilter",
"level": "ERROR",
"appender": {
"type": "smtp",
"recipients": "test@test.com",
"sendInterval": 10, //sec
"transport": "SMTP",
"SMTP": {
"host": "smtp.gmail.com",
"secureConnection": false,
"port": 587,
"auth": {
"user": "***********",
"pass": "***********"
},
"debug": true
}
}
});
/*config.log4js.appenders.push({
"type": "logLevelFilter",
"level": "ERROR",
"appender": {
"type": "smtp",
"recipients": "***********",
"sendInterval": 10, //sec
"transport": "SMTP",
"SMTP": {
"host": "smtp.gmail.com",
"secureConnection": false,
"port": 587,
"auth": {
"user": "***********",
"pass": "***********"
},
"debug": true
}
}
});*/
config.availableWappers.TOKENIZER.commandTemplate = 'python /home/priit/Programs/KEELELIIN/pyutil/tokenizer.py -i [data] -o [outputPath1]';
config.service.staticParams.storagePath = "/home/priit/wrapper";
config.service.staticParams.tmpPath = "/tmp/wrapper/";
module.exports = config;*/
//config.wrapper = config.availableWappers.<WRAPPERKEY>; //eg config.wrapper = config.availableWappers.TOKENIZER;
//config.wrapper.command.commandTemplate = 'python /home/priit/Programs/KEELELIIN/pyutil/tokenizer.py -i [data] -o [outputPath1]';
//config.fs.storagePath = "/home/priit/wrapper";
//config.fs.tmpPath = "/tmp/wrapper/";
module.exports = config;
\ No newline at end of file
......@@ -7,8 +7,7 @@ router.use('/api/v1/service', require(__base + 'controllers/api/v1/service'));
/* GET home page. Service description from config */
router.get('/', function( req, res ) {
res.render('index', {
title: config.service.title,
description: config.service.description
title: config.wrapper.title
});
});
......
......@@ -25,13 +25,13 @@ function ServiceRequest( requestBody, requestFiles ) {
this._mapParams = function(){
var staticParams = config.service.staticParams;
var staticParams = config.wrapper.requestConf.staticParams;
for(var property in config.service.requestBodyTemplate){
for(var property in config.wrapper.requestConf.requestBodyTemplate){
var value = self.data[property];
var mapping = config.service.requestBodyParamsMappings[property];
var mapping = config.wrapper.requestConf.requestBodyParamsMappings[property];
if(staticParams[property] != undefined){
value = staticParams[property];
......@@ -57,7 +57,7 @@ function ServiceRequest( requestBody, requestFiles ) {
this._checkFiles = function () {
for( var fileId in config.service.requestFiles){
for( var fileId in config.wrapper.requestConf.requestFiles ){
var file = this.files[fileId];
if(!file){
self.setMessage(fileId, 'Nõutud faili ei saadetud');
......
......@@ -14,7 +14,7 @@ var CleanerService = function () {
this.init = function(){
logger.debug('init');
if(!config.service.staticParams.sessionMaxLifetime){
if(!config.wrapper.requestConf.staticParams.sessionMaxLifetime){
return;
}
......@@ -27,7 +27,7 @@ var CleanerService = function () {
this.cleanSystem = function () {
logger.debug('clean A');
var folder = config.service.staticParams.storagePath;
var folder = config.fs.storagePath;
fs.readdir(folder, function(err, files) {
console.log('readFolder');
......@@ -77,7 +77,7 @@ var CleanerService = function () {
var timeModified = stat.mtime.getTime();
var currentTime = new Date().getTime();
if(currentTime - timeModified > ( config.service.staticParams.sessionMaxLifetime * 1000 )){
if(currentTime - timeModified > ( config.wrapper.requestConf.staticParams.sessionMaxLifetime * 1000 )){
sessionService.removeSession(file, function (err) {
if(err){
logger.error('Session delete failed: ' + file);
......
......@@ -5,7 +5,7 @@ var config = require(__base + 'config');
var DaoService = function(){
var self = this;
var prefix = config.service.staticParams.uniqueId + ':';
var prefix = config.wrapper.port + ':';
this.client = redis.createClient(config.redis.port, config.redis.host, {});
......
var logger = require('log4js').getLogger('local_executor');
var config = require('../../../config');
var sessionService = require('../../service/sessionService');
var Session = require('../../model/session');
var LocalCommand = require('../../mapper/localCommand');
var spawn = require('child_process').spawn;
var fs = require('fs');
......
......@@ -77,7 +77,7 @@ function SessionService() {
};
this.getStorePath = function (sessionId) {
return config.service.staticParams.storagePath + '/' + sessionId;
return config.fs.storagePath + '/' + sessionId;
};
this.storeToFile = function (sessionId, value, options, callback) {
......
var logger = require('log4js').getLogger('wrapper_service');
var config = require('../../config');
var Processor = require(__base + '/wrapper/' + config.service.staticParams.wrapper);
var Processor = require(__base + '/wrapper/' + config.wrapper.class);
var sessionService = require('./sessionService');
var Session = require('../model/session');
function WrapperService() {
var self = this;
......
......@@ -2,4 +2,4 @@ extends layout
block content
h1= title
p= description
var logger = require('log4js').getLogger('wrapper');
var config = require('./../config');
var localExecutor = require('./../src/service/executor/localExecutor');
var Session = require('../src/model/session');
var CommandModel = require('../src/mapper/commandModel');
var fs = require('fs');
var mime = require('mime');
var path = require('path');
function Morfyhestaja(){
var self = this;
this.process = function ( session, callback) {
self.getCommandModel(session, function (err, model) {
logger.debug('GetCommandModel callback');
if(err) return callback(err);
localExecutor.execute( model, function ( err, response ) {
if(err) return callback(err);
logger.debug('Program is finished');
if(response.isSuccess){
session.message = Session.messages.OK;
} else {
session.setErrors(response.errors);
}
session.addOutputFile('output', {
type: 'output',
fileName: path.basename(response.stdOutPath),
filePath: response.stdOutPath,
contentType: mime.lookup(response.stdOutPath)
});
return callback( err, session );
});
});
};
this.getCommandModel = function (session, callback) {
var model = new CommandModel();
model.serviceProperties.commandTemplate = config.availableCommands.MORFANALYZER.commandTemplate;
model.init( session );
model.setTextValue('data', session.requestFiles.content);
model.render(function (err) {
logger.debug('Render callback');
callback(err, model);
});
}
}
module.exports = Morfyhestaja;
\ No newline at end of file
var logger = require('log4js').getLogger('wrapper');
var config = require('./../config');
var localExecutor = require('./../src/service/executor/localExecutor');
var Session = require('../src/model/session');
var CommandModel = require('../src/mapper/commandModel');
var fs = require('fs');
var mime = require('mime');
var path = require('path');
function Osalausestaja(){
var self = this;
this.process = function ( session, callback) {
self.getCommandModel(session, function (err, model) {
logger.debug('GetCommandModel callback');
if(err) return callback(err);
localExecutor.execute( model, function ( err, response ) {
if(err) return callback(err);
logger.debug('Program is finished');
if(response.isSuccess){
session.message = Session.messages.OK;
} else {
session.setErrors(response.errors);
}
session.addOutputFile('output', {
type: 'output',
fileName: path.basename(response.stdOutPath),
filePath: response.stdOutPath,
contentType: mime.lookup(response.stdOutPath)
});
return callback( err, session );
});
});
};
this.getCommandModel = function (session, callback) {
var model = new CommandModel();
model.serviceProperties.commandTemplate = config.availableCommands.OSALAUSESTAJA.commandTemplate;
model.init( session );
model.setTextValue('data', session.requestFiles.content);
model.render(function (err) {
logger.debug('Render callback');
callback(err, model);
});
}
}
module.exports = Osalausestaja;
\ No newline at end of file
......@@ -7,7 +7,7 @@ var fs = require('fs');
var mime = require('mime');
var path = require('path');
function Lausestaja(){
function SimpleLocalCommand(){
var self = this;
......@@ -41,7 +41,7 @@ function Lausestaja(){
this.getCommandModel = function (session, callback) {
var model = new CommandModel();
model.serviceProperties.commandTemplate = config.availableCommands.LAUSESTAJA.commandTemplate;
model.serviceProperties.commandTemplate = config.wrapper.command.commandTemplate;
model.init( session );
model.setTextValue('data', session.requestFiles.content);
model.render(function (err) {
......@@ -51,4 +51,4 @@ function Lausestaja(){
}
}
module.exports = Lausestaja;
\ No newline at end of file
module.exports = SimpleLocalCommand;
\ No newline at end of file
......@@ -99,7 +99,11 @@ function Tokenizer(){
this.getCommandModel = function (session, callback) {
var model = new CommandModel();
model.serviceProperties.commandTemplate = config.availableCommands.TOKENIZER.commandTemplate;
logger.error('commandTemplate');
logger.error(config.wrapper);
model.serviceProperties.commandTemplate = config.wrapper.command.commandTemplate;
model.init( session );
model.setTextValue('data', session.requestFiles.content);
model.addOutputPath('outputPath1', {extension: 'txt'});
......
var config = require('./global');
config.port = 3004;
config.service.title = 'Arhiivi lahtipakkija';
config.service.staticParams.uniqueId = 'unzipper';
config.service.staticParams.wrapper = config.availableWappers.ARCHIVE_EXTRACTOR.class;
module.exports = config;
\ No newline at end of file
var config = {};
config.port = 3000;
config.redis = {
host: "127.0.0.1",
port: 6379
};
config.fs = {
storagePath: "/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/tmp",
tmpPath: "/tmp/wrapper/"
};
config.paramUsageTypes = {
META: 'meta', //ei kasutata utiliidi parameetrina
STRING : 'string', //parameeter asendatakse väärtusega
FILE : 'file' //parameeteri väärtus salvestatakse faili ja faili pathi kasutatakse argumendina
};
config.wrapper = {
title: null,
port: null,
class: null,
command: null,
requestConf: null
};
config.log4js = {
appenders: [
{ type: 'console',
layout: {
type: 'pattern',
pattern: "[%d] %[[%x{pid}] [%5.5p]%] %c - %m",
pattern: "[%d] %[[%x{port}-%x{pid}][%5.5p]%] %c - %m",
tokens: {
pid: process.pid
pid: process.pid,
port: function () {
return config.wrapper.port;
}
}
}
},
......@@ -45,56 +65,11 @@ config.log4js = {
]
};
config.availableWappers = {
TOKENIZER : {
class: 'tokenizer'
},
MORFANALYZER : {
class: 'morfanalyzer'
},
LAUSESTAJA : {
class: 'lausestaja'
},
OSALAUSESTAJA : {
class: 'osalausestaja'
},
ARCHIVE_EXTRACTOR: {
class: 'archiveExtractor'
}
};
config.availableCommands = {
TOKENIZER : {
commandTemplate: 'python /var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/tokenizer/tokenizer.py -i [data] -o [outputPath1]'
},
MORFANALYZER : {
commandTemplate: '/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./morfanalyzer.sh [data]'
},
LAUSESTAJA : {
commandTemplate: '/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./lausestaja.sh [data]'
},
OSALAUSESTAJA : {
commandTemplate: '/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./osalausestaja.sh [data]'
}
};
config.paramUsageTypes = {
META: 'meta', //ei kasutata utiliidi parameetrina
STRING : 'string', //parameeter asendatakse väärtusega
FILE : 'file' //parameeteri väärtus salvestatakse faili ja faili pathi kasutatakse argumendina
};
config.service = {
title: 'Teenus',
description: 'Teenuse kirjeldus',
//Siia peab olema märgitud kõik võimalikud parameetrid va sisendfailid
/*var someCommandRequest = {
requestBodyTemplate: {
is_async: null//,
//someparam: null
someparam: null
},
//Iga parameeter peab ka siin esindatud olema
requestBodyParamsMappings: {
is_async: {
usageType: config.paramUsageTypes.META,
......@@ -104,34 +79,138 @@ config.service = {
required: true,
allowEmpty: false,
validator: function(value, request){ return true; }
}/*,
},
someparam: {
usageType: config.paramUsageTypes.STRING,
filter: function(value){
if(value == 'yes'){
return '-o';
}
return null;
if(value == 'yes'){
return '-o';
}
return null;
},
required: false,
allowEmpty: true,
validator: function(value, request){ return true; }
}*/
validator: function(value, request){
return true;
}
}
},
requestFiles: {
content: null
},
staticParams: {
uniqueId: 'wrapper',
wrapper: config.availableWappers.TOKENIZER.class,
storagePath: "/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/tmp",
tmpPath: "/tmp/wrapper/",
sessionMaxLifetime: 600, //(s) Sessioonid, mille failide viimased muudatused on vanemad kui antud aeg, kustutatakse süsteemist
//siin saab päringu parameetrite väärtused üle kirjutada
isAsync: undefined //väärtustega true/false saab päringu väärtuse üle kirjutada
}
};*/
var simpleCommandRequest = {
requestBodyTemplate: {
is_async: null
},
requestBodyParamsMappings: {
is_async: {
usageType: config.paramUsageTypes.META,
filter: function(value){
return value == 1;
},
required: true,
allowEmpty: false,
validator: function(value, request){ return true; }
}
},
requestFiles: {
content: null
},
staticParams: {
sessionMaxLifetime: 600,
isAsync: undefined
}
};
config.availableCommands = {
TOKENIZER : {
commandTemplate: 'python /var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/tokenizer/tokenizer.py -i [data] -o [outputPath1]'
},
MORFANALYSAATOR : {
commandTemplate: '/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./morfanalyzer.sh [data]'
},
LAUSESTAJA : {
commandTemplate: '/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./lausestaja.sh [data]'
},
OSALAUSESTAJA : {
commandTemplate: '/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./osalausestaja.sh [data]'
},
MORFYHESTAJA : {
commandTemplate: '/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./morfyhestaja.sh [data]'
},
PIND_SYN : {
commandTemplate: '/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./pindsyn.sh [data]'
},
S6LT_SYN : {
commandTemplate: '/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./s6ltsyn.sh [data]'
}
};
config.availableWappers = {
TOKENIZER : {
title: 'Sõnestaja',
port: 3008,
class: 'tokenizer',
command: config.availableCommands.TOKENIZER,
requestConf: simpleCommandRequest
},
MORFANALYSAATOR : {
title: 'Morfoloogiline analüüs',
port: 3002,
class: 'simpleLocalCommand',
command: config.availableCommands.MORFANALYSAATOR,
requestConf: simpleCommandRequest
},
LAUSESTAJA : {
title: 'Lausestaja',
port: 3001,
class: 'simpleLocalCommand',
command: config.availableCommands.LAUSESTAJA,
requestConf: simpleCommandRequest
},
OSALAUSESTAJA : {
title: 'Osalausestamine',
port: 3003,
class: 'simpleLocalCommand',
command: config.availableCommands.OSALAUSESTAJA,
requestConf: simpleCommandRequest
},
ARCHIVE_EXTRACTOR: {
title: 'Arhiivi lahtipakkija',
port: 3007,
class: 'archiveExtractor',
requestConf: simpleCommandRequest
},
MORFYHESTAJA: {
title: 'Morfoloogiline ühestamine (kitsenduste grammatika)',
port: 3004,
class: 'simpleLocalCommand',
command: config.availableCommands.MORFYHESTAJA,
requestConf: simpleCommandRequest
},
PIND_SYN: {
title: 'Pindsüntaktiline analüüs',
port: 3005,
class: 'simpleLocalCommand',
command: config.availableCommands.PIND_SYN,
requestConf: simpleCommandRequest
},
S6LT_SYN: {
title: 'Sõltuvussüntaktiline analüüs (ja järeltöötlus)',
port: 3006,
class: 'simpleLocalCommand',
command: config.availableCommands.S6LT_SYN,
requestConf: simpleCommandRequest
}
};
module.exports = config;
\ No newline at end of file
var config = require('./global');
config.port = 3005;
config.service.title = 'Lausestaja';
config.service.staticParams.uniqueId = 'sentence';
config.service.staticParams.wrapper = config.availableWappers.LAUSESTAJA.class;
module.exports = config;
\ No newline at end of file
var config = require('./global');
config.port = 3003;
config.service.title = 'Morfoloogiline analüüs';
config.service.staticParams.uniqueId = 'morfanalyzer';
config.service.staticParams.wrapper = config.availableWappers.MORFANALYZER.class;
module.exports = config;
\ No newline at end of file
var config = require('./global');
config.port = 3005;
config.service.title = 'Osalausestaja';
config.service.staticParams.uniqueId = 'subsentence';
config.service.staticParams.wrapper = config.availableWappers.OSALAUSESTAJA.class;
module.exports = config;
\ No newline at end of file