Skip to content

GitLab

K
keeleliin-wrapper
Commit 856f8d15 authored by priit's avatar priit
Browse files
Options

Dockeri alge

parent ceae0926
Hide whitespace changes
Inline Side-by-side
Showing with 185 additions and 327 deletions
.gitignore
View file @ 856f8d15
......@@ -11,5 +11,5 @@ tmp_tests/94megafile
tmp_tests/189megafile
keeleliin.log
coverage
wrapper_configs/local_global.js
global_config/local_global.js
updateWrappers.sh
\ No newline at end of file
Dockerfile
View file @ 856f8d15
FROM ubuntu:14.04
# docker build -t kl_wrapper .
# docker run --name kl_redis --rm --restart=no redis
# docker run --name kl_wrapper -d -it --link kl_redis:redis -p 3000:3000 -v /config:/config -v /logs:/logs kl_wrapper
# docker kill kl_wrapper
# docker rm kl_wrapper
#
MAINTAINER priit@bitweb.ee
FROM ubuntu:14.04
RUN apt-get update && \
apt-get -y install curl && \
curl -sL https://deb.nodesource.com/setup | sudo bash - && \
apt-get -y install python build-essential nodejs
RUN apt-get update && apt-get -y install nodejs npm git
# Provides cached layer for node_modules
ADD package.json /tmp/package.json
RUN cd /tmp && npm install && npm install -g forever
RUN mkdir -p /src && cp -a /tmp/node_modules /src/
RUN npm install -g forever
# Define working directory
WORKDIR /src
ADD . /src
RUN mkdir -p /src
RUN cd /src && git clone 'https://github.com/BitWeb/keeleliin-wrapper-service.git' . && echo "Git is cloned 1"
RUN cd /src && npm install && echo "NPM is installed 1"
# Expose port
#Expose port
EXPOSE 3000
VOLUME ["/wrapper"]
VOLUME ["/src"]
# Run app using forever
CMD ["forever", "/src/app.js"]
\ No newline at end of file
CMD /./src/docker_start.sh
\ No newline at end of file
config_dist.js
View file @ 856f8d15
var config = require('./wrapper_configs/global');
var config = require('./global_config/global');
//config.serverUrl = 'http://localhost';
//config.redis = {
// host: process.env.REDIS_PORT_6379_TCP_ADDR || "127.0.0.1",
// port: process.env.REDIS_PORT_6379_TCP_PORT || 6379
//};
//config.integration = [
// {
// installUrl: 'http://keeleliin.keeleressursid.ee:3000/api/v1/service/install',
// apiKey: 'server-wrapper-api-key'
// }
//];
//config.fs = {
// storagePath: "/wrapper/files",
// tmpPath: "/wrapper/tmp"
//};
/*config.log4js.appenders.push({
"type": "logLevelFilter",
......@@ -30,10 +50,52 @@ var config = require('./wrapper_configs/global');
}
});*/
config.serverUrl = process.env.SERVER_URL || 'http://dev.bitweb.ee';
config.wrapper = process.env.WRAPPER ? (config.availableWrappers + process.env.WRAPPER) : config.availableWrappers.CONCAT;
config.wrapper = {
id: 'concat', // unikaalne lühinimi
title: 'Lihtne konjateneerija', //Avalik nimi
description: 'Konkateneerib etteantud failid üheks suureks failiks', //Kirjeldus
port: 3000, //port
class: 'simpleLocalCommand', //wrapperi failinimi wrapper kaustast, mida utiliidi käivitamiseks kasutatakse
command: 'cat [data]', // utiliidi käsurea käsk
requestConf: { //Päringu seadistus
requestBodyParamsMappings: {
isAsync: {
type: config.paramTypes.SELECT,
options: ['0', '1'],
value: '1',
usageType: config.paramUsageTypes.META,
filter: function (value) {
return value == 1;
},
required: true,
allowEmpty: false,
validator: function (value, request) {
if( typeof value != 'boolean' ){
request.setMessage('isAsync', 'Peab olema boolean');
return false;
}
return true;
}
}
},
requestFiles: {
content: {
type: 'text',
sizeLimit: 0,
sizeUnit: 'byte',
isList: true
}
}
},
outputTypes: [ //teenuse väljundressursside kirjldused
{
type: 'text',
key: 'output'
}
],
sessionMaxLifetime: 600 // sessiooni maksimaalne kestvus
};
config.fs.storagePath = "/files";
config.fs.tmpPath = "/tmp";
module.exports = config;
\ No newline at end of file
controllers/api/v1/service.js
View file @ 856f8d15
......@@ -13,9 +13,6 @@ router.get('/config', function(req, res) {
router.post('/', function ( req, res ) {
logger.debug(req.body);
logger.debug(req.files);
var serviceRequest = new ServiceRequest( req.body, req.files );
if(!serviceRequest.isValid()){
......@@ -23,6 +20,8 @@ router.post('/', function ( req, res ) {
return;
}
logger.debug(serviceRequest );
wrapperService.execute( serviceRequest, function (err, data) {
if(err) return res.send({errors: err});
res.send(data);
......
docker_start.sh
View file @ 856f8d15
#!/bin/bash
if [ ! -f /config/config.js ]; then
cp -R -u -p /src/config_dist.js /config/config.js
fi
if [ ! -f /src/config.js ]; then
ln -s /config/config.js /src/config.js
fi
if [ ! -f /config/wrapper_configs/global.js ]; then
mkdir -p /config/wrapper_configs
ln -s /src/wrapper_configs/global.js /config/wrapper_configs/global.js
cp -R -u -p /src/config_dist.js /src/config.js
fi
forever /src/app.js
\ No newline at end of file
wrapper_configs/global.js → global_config/global.js
View file @ 856f8d15
......@@ -5,13 +5,9 @@ config.redis = {
port: process.env.REDIS_PORT_6379_TCP_PORT || 6379
};
config.serverUrl = 'http://dev.bitweb.ee';
config.serverUrl = 'http://localhost';
config.integration = [
{
installUrl: 'http://dev.bitweb.ee:3000/api/v1/service/install',
apiKey: 'server-wrapper-api-key'
},
{
installUrl: 'http://keeleliin.keeleressursid.ee:3000/api/v1/service/install',
apiKey: 'server-wrapper-api-key'
......@@ -23,6 +19,11 @@ config.fs = {
tmpPath: "/wrapper/tmp"
};
config.paramTypes = {
TEXT: 'text',
SELECT: 'select'
};
config.paramUsageTypes = {
META: 'meta', //ei kasutata utiliidi parameetrina
STRING: 'string', //parameeter asendatakse väärtusega
......@@ -33,7 +34,7 @@ config.log4js = {
appenders: [
{
"type": "logLevelFilter",
"level": "ERROR",
"level": "DEBUG",
"appender": {
type: 'console',
layout: {
......@@ -53,7 +54,7 @@ config.log4js = {
"level": "ERROR",
"appender": {
type: 'file',
filename: '/logs/wrapper.log',
filename: __dirname + '/../wrapper.log',
layout: {
type: 'pattern',
pattern: "[%d] %[[%x{port}-%x{pid}][%5.5p]%] %c - %m",
......@@ -69,48 +70,8 @@ config.log4js = {
]
};
config.wrapper = {
id: null, // unique service identifier
title: null,
description: '',
port: null,
class: null,
command: null,
requestConf: null
};
config.availableCommands = {
TOKENIZER: {
commandTemplate: 'python /wrapper/utils/picoutils/tokenizer.py -i [data] -o [output]'
},
MORFANALYSAATOR: {
commandTemplate: '/wrapper/utils/./morfanalyzer.sh [data]'
},
LAUSESTAJA: {
commandTemplate: '/wrapper/utils/./lausestaja.sh [data]'
},
OSALAUSESTAJA: {
commandTemplate: '/wrapper/utils/./osalausestaja.sh [data]'
},
MORFYHESTAJA: {
commandTemplate: '/wrapper/utils/./morfyhestaja.sh [data]'
},
PIND_SYN: {
commandTemplate: '/wrapper/utils/./pindsyn.sh [data]'
},
S6LT_SYN: {
commandTemplate: '/wrapper/utils/./s6ltsyn.sh [data]'
},
CONCAT: {
commandTemplate: 'cat [data]'
},
MORPH_TAGGER: {
commandTemplate: 'python /wrapper/utils/picoutils/morph_tagger.py -i [data] -o [output]'
}
};
var isAsyncParamValue = {
type: 'select',
config.isAsyncParamDescription = {
type: config.paramTypes.SELECT,
options: ['0', '1'],
value: '1',
usageType: config.paramUsageTypes.META,
......@@ -124,8 +85,8 @@ var isAsyncParamValue = {
}
};
var stringKeyValue = {
type: 'text',
config.stringKeyDescription = {
type: config.paramTypes.TEXT,
value: undefined,
usageType: config.paramUsageTypes.STRING,
filter: null,
......@@ -136,8 +97,8 @@ var stringKeyValue = {
}
};
var fileKeyValue = {
type: 'text',
config.fileKeyDescription = {
type: config.paramTypes.TEXT,
value: undefined,
usageType: config.paramUsageTypes.FILE,
filter: null,
......@@ -148,18 +109,62 @@ var fileKeyValue = {
}
};
config.wrapper = {
id: null, // unikaalne lühinimi
title: null, //Avalik nimi
description: '', //Kirjeldus
port: null, //port
class: null, //wrapperi failinimi wrapper kaustast, mida utiliidi käivitamiseks kasutatakse
command: null, // utiliidi käsurea käsk
requestConf: null, //Päringu seadistus
outputTypes: null, //teenuse väljundressursside kirjldus
sessionMaxLifetime: 600 // sessiooni maksimaalne kestvus
};
config.availableCommands = {
TOKENIZER: {
commandTemplate: 'python /var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/picoutils/tokenizer.py -i [data] -o [output]'
},
MORFANALYSAATOR: {
commandTemplate: '/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./morfanalyzer.sh [data]'
},
LAUSESTAJA: {
commandTemplate: '/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./lausestaja.sh [data]'
},
OSALAUSESTAJA: {
commandTemplate: '/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./osalausestaja.sh [data]'
},
MORFYHESTAJA: {
commandTemplate: '/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./morfyhestaja.sh [data]'
},
PIND_SYN: {
commandTemplate: '/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./pindsyn.sh [data]'
},
S6LT_SYN: {
commandTemplate: '/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./s6ltsyn.sh [data]'
},
CONCAT: {
commandTemplate: 'cat [data]'
},
MORPH_TAGGER: {
commandTemplate: 'python /var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/picoutils/morph_tagger.py -i [data] -o [output]'
}
};
config.availableWrappers = {
LAUSESTAJA: {
title: 'Lausestaja',
description: '',
id: 'lau',
port: process.env.SERVER_PORT || 3001,
port: 3001,
class: 'simpleLocalCommand',
command: config.availableCommands.LAUSESTAJA,
requestConf: {
requestBodyParamsMappings: {
isAsync: isAsyncParamValue
isAsync: config.isAsyncParamDescription
},
requestFiles: {
content: {
......@@ -182,12 +187,12 @@ config.availableWrappers = {
title: 'Morfoloogiline analüüs',
description: '',
id: 'moa',
port: process.env.SERVER_PORT || 3002,
port: 3002,
class: 'simpleLocalCommand',
command: config.availableCommands.MORFANALYSAATOR,
requestConf: {
requestBodyParamsMappings: {
isAsync: isAsyncParamValue
isAsync: config.isAsyncParamDescription
},
requestFiles: {
content: {
......@@ -210,12 +215,12 @@ config.availableWrappers = {
title: 'Osalausestaja',
description: '',
id: 'osl',
port: process.env.SERVER_PORT || 3003,
port: 3003,
class: 'simpleLocalCommand',
command: config.availableCommands.OSALAUSESTAJA,
requestConf: {
requestBodyParamsMappings: {
isAsync: isAsyncParamValue
isAsync: config.isAsyncParamDescription
},
requestFiles: {
content: {
......@@ -238,12 +243,12 @@ config.availableWrappers = {
title: 'Morfoloogiline ühestamine (kitsenduste grammatika)',
description: '',
id: 'moy',
port: process.env.SERVER_PORT || 3004,
port: 3004,
class: 'simpleLocalCommand',
command: config.availableCommands.MORFYHESTAJA,
requestConf: {
requestBodyParamsMappings: {
isAsync: isAsyncParamValue
isAsync: config.isAsyncParamDescription
},
requestFiles: {
content: {
......@@ -266,12 +271,12 @@ config.availableWrappers = {
title: 'Pindsüntaktiline analüüs',
description: '',
id: 'pia',
port: process.env.SERVER_PORT || 3005,
port: 3005,
class: 'simpleLocalCommand',
command: config.availableCommands.PIND_SYN,
requestConf: {
requestBodyParamsMappings: {
isAsync: isAsyncParamValue
isAsync: config.isAsyncParamDescription
},
requestFiles: {
content: {
......@@ -294,12 +299,12 @@ config.availableWrappers = {
title: 'Sõltuvussüntaktiline analüüs (ja järeltöötlus)',
description: '',
id: 's6a',
port: process.env.SERVER_PORT || 3006,
port: 3006,
class: 'simpleLocalCommand',
command: config.availableCommands.S6LT_SYN,
requestConf: {
requestBodyParamsMappings: {
isAsync: isAsyncParamValue
isAsync: config.isAsyncParamDescription
},
requestFiles: {
content: {
......@@ -323,11 +328,11 @@ config.availableWrappers = {
title: 'Arhiivi lahtipakkija',
description: '',
id: 'uzip',
port: process.env.SERVER_PORT || 3007,
port: 3007,
class: 'archiveExtractor',
requestConf: {
requestBodyParamsMappings: {
isAsync: isAsyncParamValue
isAsync: config.isAsyncParamDescription
},
requestFiles: {
content: {
......@@ -350,12 +355,12 @@ config.availableWrappers = {
title: 'Sõnestaja pipe',
description: '',
id: 'tok',
port: process.env.SERVER_PORT || 3008,
port: 3008,
class: 'inputOutputLocalCommand',
command: config.availableCommands.TOKENIZER,
requestConf: {
requestBodyParamsMappings: {
isAsync: isAsyncParamValue
isAsync: config.isAsyncParamDescription
},
requestFiles: {
content: {
......@@ -377,12 +382,12 @@ config.availableWrappers = {
CONCAT: {
title: 'Lihtne konkateneerija',
id: 'concat',
port: process.env.SERVER_PORT || 3009,
port: 3009,
class: 'simpleLocalCommand',
command: config.availableCommands.CONCAT,
requestConf: {
requestBodyParamsMappings: {
isAsync: isAsyncParamValue
isAsync: config.isAsyncParamDescription
},
requestFiles: {
content: {
......@@ -404,12 +409,12 @@ config.availableWrappers = {
MORPH_TAGGER: {
title: 'Morfoloogiline ühestaja pipe',
id: 'tag',
port: process.env.SERVER_PORT || 3010,
port: 3010,
class: 'inputOutputLocalCommand',
command: config.availableCommands.MORPH_TAGGER,
requestConf: {
requestBodyParamsMappings: {
isAsync: isAsyncParamValue
isAsync: config.isAsyncParamDescription
},
requestFiles: {
content: {
......
src/model/serviceRequest.js
View file @ 856f8d15
......@@ -44,7 +44,7 @@ function ServiceRequest( requestBody, requestFiles ) {
continue;
}
if(!value){
if(value === null || value === undefined){
value = mapping.value;
}
......
wrapper/inputOutputLocalCommand.js
View file @ 856f8d15
......@@ -13,41 +13,33 @@ function InputOutputLocalCommand(){
var self = this;
this.process = function ( session, callback) {
var contentFile = session.requestFiles.content;
fs.readFile(contentFile, 'utf-8', function (err, sourceText) {
self._getCommandModel(session, function (err, model) {
logger.debug('getCommandModel callback');
if(err) return callback(err);
if(err){
session.setErrors(err);
return callback( err, session );
}
localExecutor.execute( model, session, function ( err, response ) {
self._getCommandModel(session, function (err, model) {
logger.debug('getCommandModel callback');
if(err) return callback(err);
localExecutor.execute( model, session, function ( err, response ) {
if(err) return callback(err);
logger.debug('Program is finished');
if(response.isSuccess){
session.message = Session.messages.OK;
} else {
session.setErrors(response.errors);
}
session.addOutputFile('output', {
type: 'output',
fileName: config.wrapper.id + '_output.' + FileUtil.getExtension( model.outputPaths.output ),
filePath: model.outputPaths.output,
contentType: mime.lookup(model.outputPaths.output)
});
return callback( err, session );
logger.debug('Program is finished');
if(response.isSuccess){
session.message = Session.messages.OK;
} else {
session.setErrors(response.errors);
}
session.addOutputFile('output', {
type: 'output',
fileName: config.wrapper.id + '_output.' + FileUtil.getExtension( model.outputPaths.output ),
filePath: model.outputPaths.output,
contentType: mime.lookup(model.outputPaths.output)
});
return callback( err, session );
});
});
};
this._getCommandModel = function (session, callback) {
......
wrapper_dockers/lausestaja/Dockerfile deleted 100644 → 0
View file @ ceae0926
# docker build -t kl_lausestaja .
#
# docker run --name kl_redis --rm --restart=no redis
#
# docker run --name kl_lausestaja --link kl_redis:redis -p 3001:3001 -v /wrapper/lausestaja/config:/config kl_lausestaja
#
# docker kill kl_lausestaja
#
# docker rm kl_lausestaja
#
#
FROM ubuntu:14.04
ENV TERM xterm
ENV WRAPPER=LAUSESTAJA
ENV SERVER_URL="http://dev.bitweb.ee"
ENV SERVER_PORT=3001
RUN apt-get update && \
apt-get -y install curl && \
curl -sL https://deb.nodesource.com/setup | sudo bash - && \
apt-get -y install python build-essential nodejs && \
apt-get -y install git && \
apt-get -y install unzip
RUN npm install -g forever
RUN mkdir -p /src
RUN mkdir -p /tmp
RUN mkdir -p /files
RUN mkdir -p /logs
RUN mkdir -p /config
RUN cd /src && git clone 'https://github.com/BitWeb/keeleliin-wrapper-service.git' . && npm install && echo "Run is Done"
RUN mkdir /utils
COPY ./utils /utils
RUN unzip /utils/CGParser.zip -d /utils/
RUN unzip /utils/morfyhestaja.zip -d /utils/
RUN chmod 777 -Rf /utils/*
#Expose port
EXPOSE $SERVER_PORT
VOLUME ["/tmp","/files","/logs", "/config"]
CMD /./src/docker_start.sh
\ No newline at end of file
wrapper_dockers/lausestaja/utils/lausestaja.sh deleted 100644 → 0
View file @ ceae0926
#!/bin/sh
BASEDIR=$(dirname $0)
RADA=$BASEDIR/morfyhestaja/konverter
cat $1 | $RADA/rlausestaja.pl
wrapper_dockers/morf_analysaator/Dockerfile deleted 100644 → 0
View file @ ceae0926
FROM ubuntu:14.04
MAINTAINER priit@bitweb.ee
RUN apt-get update && \
apt-get -y install curl && \
curl -sL https://deb.nodesource.com/setup | sudo bash - && \
apt-get -y install python build-essential nodejs && \
apt-get -y install git
RUN npm install -g forever
RUN mkdir -p /src
RUN mkdir -p /tmp
RUN mkdir -p /files
RUN mkdir -p /logs
RUN mkdir -p /config
RUN cd /src && git clone 'https://github.com/BitWeb/keeleliin-wrapper-service.git' . && npm install && echo "RunisDon"
// CGParser
// vabamorf