Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
keeleliin
keeleliin-wrapper
Commits
856f8d15
Commit
856f8d15
authored
Nov 04, 2015
by
priit
Browse files
Dockeri alge
parent
ceae0926
Changes
23
Hide whitespace changes
Inline
Side-by-side
.gitignore
View file @
856f8d15
...
...
@@ -11,5 +11,5 @@ tmp_tests/94megafile
tmp_tests/189megafile
keeleliin.log
coverage
wrapper
_config
s
/local_global.js
global
_config/local_global.js
updateWrappers.sh
\ No newline at end of file
Dockerfile
View file @
856f8d15
FROM
ubuntu:14.04
# docker build -t kl_wrapper .
# docker run --name kl_redis --rm --restart=no redis
# docker run --name kl_wrapper -d -it --link kl_redis:redis -p 3000:3000 -v /config:/config -v /logs:/logs kl_wrapper
# docker kill kl_wrapper
# docker rm kl_wrapper
#
MAINTAINER
priit@bitweb.ee
FROM
ubuntu:14.04
RUN
apt-get update
&&
\
apt-get
-y
install
curl
&&
\
curl
-sL
https://deb.nodesource.com/setup |
sudo
bash -
&&
\
apt-get
-y
install
python build-essential nodejs
RUN
apt-get update
&&
apt-get
-y
install
nodejs npm git
# Provides cached layer for node_modules
ADD
package.json /tmp/package.json
RUN
cd
/tmp
&&
npm
install
&&
npm
install
-g
forever
RUN
mkdir
-p
/src
&&
cp
-a
/tmp/node_modules /src/
RUN
npm
install
-g
forever
# Define working directory
WORKDIR
/src
ADD
. /src
RUN
mkdir
-p
/src
RUN
cd
/src
&&
git clone
'https://github.com/BitWeb/keeleliin-wrapper-service.git'
.
&&
echo
"Git is cloned 1"
RUN
cd
/src
&&
npm
install
&&
echo
"NPM is installed 1"
#
Expose port
#Expose port
EXPOSE
3000
VOLUME
["/
wrapper
"]
VOLUME
["/
src
"]
# Run app using forever
CMD
["forever", "/src/app.js"]
\ No newline at end of file
CMD
/./src/docker_start.sh
\ No newline at end of file
config_dist.js
View file @
856f8d15
var
config
=
require
(
'
./wrapper_configs/global
'
);
var
config
=
require
(
'
./global_config/global
'
);
//config.serverUrl = 'http://localhost';
//config.redis = {
// host: process.env.REDIS_PORT_6379_TCP_ADDR || "127.0.0.1",
// port: process.env.REDIS_PORT_6379_TCP_PORT || 6379
//};
//config.integration = [
// {
// installUrl: 'http://keeleliin.keeleressursid.ee:3000/api/v1/service/install',
// apiKey: 'server-wrapper-api-key'
// }
//];
//config.fs = {
// storagePath: "/wrapper/files",
// tmpPath: "/wrapper/tmp"
//};
/*config.log4js.appenders.push({
"type": "logLevelFilter",
...
...
@@ -30,10 +50,52 @@ var config = require('./wrapper_configs/global');
}
});*/
config
.
serverUrl
=
process
.
env
.
SERVER_URL
||
'
http://dev.bitweb.ee
'
;
config
.
wrapper
=
process
.
env
.
WRAPPER
?
(
config
.
availableWrappers
+
process
.
env
.
WRAPPER
)
:
config
.
availableWrappers
.
CONCAT
;
config
.
wrapper
=
{
id
:
'
concat
'
,
// unikaalne lühinimi
title
:
'
Lihtne konjateneerija
'
,
//Avalik nimi
description
:
'
Konkateneerib etteantud failid üheks suureks failiks
'
,
//Kirjeldus
port
:
3000
,
//port
class
:
'
simpleLocalCommand
'
,
//wrapperi failinimi wrapper kaustast, mida utiliidi käivitamiseks kasutatakse
command
:
'
cat [data]
'
,
// utiliidi käsurea käsk
requestConf
:
{
//Päringu seadistus
requestBodyParamsMappings
:
{
isAsync
:
{
type
:
config
.
paramTypes
.
SELECT
,
options
:
[
'
0
'
,
'
1
'
],
value
:
'
1
'
,
usageType
:
config
.
paramUsageTypes
.
META
,
filter
:
function
(
value
)
{
return
value
==
1
;
},
required
:
true
,
allowEmpty
:
false
,
validator
:
function
(
value
,
request
)
{
if
(
typeof
value
!=
'
boolean
'
){
request
.
setMessage
(
'
isAsync
'
,
'
Peab olema boolean
'
);
return
false
;
}
return
true
;
}
}
},
requestFiles
:
{
content
:
{
type
:
'
text
'
,
sizeLimit
:
0
,
sizeUnit
:
'
byte
'
,
isList
:
true
}
}
},
outputTypes
:
[
//teenuse väljundressursside kirjldused
{
type
:
'
text
'
,
key
:
'
output
'
}
],
sessionMaxLifetime
:
600
// sessiooni maksimaalne kestvus
};
config
.
fs
.
storagePath
=
"
/files
"
;
config
.
fs
.
tmpPath
=
"
/tmp
"
;
module
.
exports
=
config
;
\ No newline at end of file
controllers/api/v1/service.js
View file @
856f8d15
...
...
@@ -13,9 +13,6 @@ router.get('/config', function(req, res) {
router
.
post
(
'
/
'
,
function
(
req
,
res
)
{
logger
.
debug
(
req
.
body
);
logger
.
debug
(
req
.
files
);
var
serviceRequest
=
new
ServiceRequest
(
req
.
body
,
req
.
files
);
if
(
!
serviceRequest
.
isValid
()){
...
...
@@ -23,6 +20,8 @@ router.post('/', function ( req, res ) {
return
;
}
logger
.
debug
(
serviceRequest
);
wrapperService
.
execute
(
serviceRequest
,
function
(
err
,
data
)
{
if
(
err
)
return
res
.
send
({
errors
:
err
});
res
.
send
(
data
);
...
...
docker_start.sh
View file @
856f8d15
#!/bin/bash
if
[
!
-f
/config/config.js
]
;
then
cp
-R
-u
-p
/src/config_dist.js /config/config.js
fi
if
[
!
-f
/src/config.js
]
;
then
ln
-s
/config/config.js /src/config.js
fi
if
[
!
-f
/config/wrapper_configs/global.js
]
;
then
mkdir
-p
/config/wrapper_configs
ln
-s
/src/wrapper_configs/global.js /config/wrapper_configs/global.js
cp
-R
-u
-p
/src/config_dist.js /src/config.js
fi
forever /src/app.js
\ No newline at end of file
wrapper
_config
s
/global.js
→
global
_config/global.js
View file @
856f8d15
...
...
@@ -5,13 +5,9 @@ config.redis = {
port
:
process
.
env
.
REDIS_PORT_6379_TCP_PORT
||
6379
};
config
.
serverUrl
=
'
http://
dev.bitweb.ee
'
;
config
.
serverUrl
=
'
http://
localhost
'
;
config
.
integration
=
[
{
installUrl
:
'
http://dev.bitweb.ee:3000/api/v1/service/install
'
,
apiKey
:
'
server-wrapper-api-key
'
},
{
installUrl
:
'
http://keeleliin.keeleressursid.ee:3000/api/v1/service/install
'
,
apiKey
:
'
server-wrapper-api-key
'
...
...
@@ -23,6 +19,11 @@ config.fs = {
tmpPath
:
"
/wrapper/tmp
"
};
config
.
paramTypes
=
{
TEXT
:
'
text
'
,
SELECT
:
'
select
'
};
config
.
paramUsageTypes
=
{
META
:
'
meta
'
,
//ei kasutata utiliidi parameetrina
STRING
:
'
string
'
,
//parameeter asendatakse väärtusega
...
...
@@ -33,7 +34,7 @@ config.log4js = {
appenders
:
[
{
"
type
"
:
"
logLevelFilter
"
,
"
level
"
:
"
ERROR
"
,
"
level
"
:
"
DEBUG
"
,
"
appender
"
:
{
type
:
'
console
'
,
layout
:
{
...
...
@@ -53,7 +54,7 @@ config.log4js = {
"
level
"
:
"
ERROR
"
,
"
appender
"
:
{
type
:
'
file
'
,
filename
:
'
/logs
/wrapper.log
'
,
filename
:
__dirname
+
'
/..
/wrapper.log
'
,
layout
:
{
type
:
'
pattern
'
,
pattern
:
"
[%d] %[[%x{port}-%x{pid}][%5.5p]%] %c - %m
"
,
...
...
@@ -69,48 +70,8 @@ config.log4js = {
]
};
config
.
wrapper
=
{
id
:
null
,
// unique service identifier
title
:
null
,
description
:
''
,
port
:
null
,
class
:
null
,
command
:
null
,
requestConf
:
null
};
config
.
availableCommands
=
{
TOKENIZER
:
{
commandTemplate
:
'
python /wrapper/utils/picoutils/tokenizer.py -i [data] -o [output]
'
},
MORFANALYSAATOR
:
{
commandTemplate
:
'
/wrapper/utils/./morfanalyzer.sh [data]
'
},
LAUSESTAJA
:
{
commandTemplate
:
'
/wrapper/utils/./lausestaja.sh [data]
'
},
OSALAUSESTAJA
:
{
commandTemplate
:
'
/wrapper/utils/./osalausestaja.sh [data]
'
},
MORFYHESTAJA
:
{
commandTemplate
:
'
/wrapper/utils/./morfyhestaja.sh [data]
'
},
PIND_SYN
:
{
commandTemplate
:
'
/wrapper/utils/./pindsyn.sh [data]
'
},
S6LT_SYN
:
{
commandTemplate
:
'
/wrapper/utils/./s6ltsyn.sh [data]
'
},
CONCAT
:
{
commandTemplate
:
'
cat [data]
'
},
MORPH_TAGGER
:
{
commandTemplate
:
'
python /wrapper/utils/picoutils/morph_tagger.py -i [data] -o [output]
'
}
};
var
isAsyncParamValue
=
{
type
:
'
select
'
,
config
.
isAsyncParamDescription
=
{
type
:
config
.
paramTypes
.
SELECT
,
options
:
[
'
0
'
,
'
1
'
],
value
:
'
1
'
,
usageType
:
config
.
paramUsageTypes
.
META
,
...
...
@@ -124,8 +85,8 @@ var isAsyncParamValue = {
}
};
var
stringKey
Value
=
{
type
:
'
text
'
,
config
.
stringKey
Description
=
{
type
:
config
.
paramTypes
.
TEXT
,
value
:
undefined
,
usageType
:
config
.
paramUsageTypes
.
STRING
,
filter
:
null
,
...
...
@@ -136,8 +97,8 @@ var stringKeyValue = {
}
};
var
fileKeyValue
=
{
type
:
'
text
'
,
config
.
fileKeyDescription
=
{
type
:
config
.
paramTypes
.
TEXT
,
value
:
undefined
,
usageType
:
config
.
paramUsageTypes
.
FILE
,
filter
:
null
,
...
...
@@ -148,18 +109,62 @@ var fileKeyValue = {
}
};
config
.
wrapper
=
{
id
:
null
,
// unikaalne lühinimi
title
:
null
,
//Avalik nimi
description
:
''
,
//Kirjeldus
port
:
null
,
//port
class
:
null
,
//wrapperi failinimi wrapper kaustast, mida utiliidi käivitamiseks kasutatakse
command
:
null
,
// utiliidi käsurea käsk
requestConf
:
null
,
//Päringu seadistus
outputTypes
:
null
,
//teenuse väljundressursside kirjldus
sessionMaxLifetime
:
600
// sessiooni maksimaalne kestvus
};
config
.
availableCommands
=
{
TOKENIZER
:
{
commandTemplate
:
'
python /var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/picoutils/tokenizer.py -i [data] -o [output]
'
},
MORFANALYSAATOR
:
{
commandTemplate
:
'
/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./morfanalyzer.sh [data]
'
},
LAUSESTAJA
:
{
commandTemplate
:
'
/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./lausestaja.sh [data]
'
},
OSALAUSESTAJA
:
{
commandTemplate
:
'
/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./osalausestaja.sh [data]
'
},
MORFYHESTAJA
:
{
commandTemplate
:
'
/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./morfyhestaja.sh [data]
'
},
PIND_SYN
:
{
commandTemplate
:
'
/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./pindsyn.sh [data]
'
},
S6LT_SYN
:
{
commandTemplate
:
'
/var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/./s6ltsyn.sh [data]
'
},
CONCAT
:
{
commandTemplate
:
'
cat [data]
'
},
MORPH_TAGGER
:
{
commandTemplate
:
'
python /var/www/bitweb.ee/keeleliin.bitweb.ee/wrapper/utils/picoutils/morph_tagger.py -i [data] -o [output]
'
}
};
config
.
availableWrappers
=
{
LAUSESTAJA
:
{
title
:
'
Lausestaja
'
,
description
:
''
,
id
:
'
lau
'
,
port
:
process
.
env
.
SERVER_PORT
||
3001
,
port
:
3001
,
class
:
'
simpleLocalCommand
'
,
command
:
config
.
availableCommands
.
LAUSESTAJA
,
requestConf
:
{
requestBodyParamsMappings
:
{
isAsync
:
isAsyncParam
Value
isAsync
:
config
.
isAsyncParam
Description
},
requestFiles
:
{
content
:
{
...
...
@@ -182,12 +187,12 @@ config.availableWrappers = {
title
:
'
Morfoloogiline analüüs
'
,
description
:
''
,
id
:
'
moa
'
,
port
:
process
.
env
.
SERVER_PORT
||
3002
,
port
:
3002
,
class
:
'
simpleLocalCommand
'
,
command
:
config
.
availableCommands
.
MORFANALYSAATOR
,
requestConf
:
{
requestBodyParamsMappings
:
{
isAsync
:
isAsyncParam
Value
isAsync
:
config
.
isAsyncParam
Description
},
requestFiles
:
{
content
:
{
...
...
@@ -210,12 +215,12 @@ config.availableWrappers = {
title
:
'
Osalausestaja
'
,
description
:
''
,
id
:
'
osl
'
,
port
:
process
.
env
.
SERVER_PORT
||
3003
,
port
:
3003
,
class
:
'
simpleLocalCommand
'
,
command
:
config
.
availableCommands
.
OSALAUSESTAJA
,
requestConf
:
{
requestBodyParamsMappings
:
{
isAsync
:
isAsyncParam
Value
isAsync
:
config
.
isAsyncParam
Description
},
requestFiles
:
{
content
:
{
...
...
@@ -238,12 +243,12 @@ config.availableWrappers = {
title
:
'
Morfoloogiline ühestamine (kitsenduste grammatika)
'
,
description
:
''
,
id
:
'
moy
'
,
port
:
process
.
env
.
SERVER_PORT
||
3004
,
port
:
3004
,
class
:
'
simpleLocalCommand
'
,
command
:
config
.
availableCommands
.
MORFYHESTAJA
,
requestConf
:
{
requestBodyParamsMappings
:
{
isAsync
:
isAsyncParam
Value
isAsync
:
config
.
isAsyncParam
Description
},
requestFiles
:
{
content
:
{
...
...
@@ -266,12 +271,12 @@ config.availableWrappers = {
title
:
'
Pindsüntaktiline analüüs
'
,
description
:
''
,
id
:
'
pia
'
,
port
:
process
.
env
.
SERVER_PORT
||
3005
,
port
:
3005
,
class
:
'
simpleLocalCommand
'
,
command
:
config
.
availableCommands
.
PIND_SYN
,
requestConf
:
{
requestBodyParamsMappings
:
{
isAsync
:
isAsyncParam
Value
isAsync
:
config
.
isAsyncParam
Description
},
requestFiles
:
{
content
:
{
...
...
@@ -294,12 +299,12 @@ config.availableWrappers = {
title
:
'
Sõltuvussüntaktiline analüüs (ja järeltöötlus)
'
,
description
:
''
,
id
:
'
s6a
'
,
port
:
process
.
env
.
SERVER_PORT
||
3006
,
port
:
3006
,
class
:
'
simpleLocalCommand
'
,
command
:
config
.
availableCommands
.
S6LT_SYN
,
requestConf
:
{
requestBodyParamsMappings
:
{
isAsync
:
isAsyncParam
Value
isAsync
:
config
.
isAsyncParam
Description
},
requestFiles
:
{
content
:
{
...
...
@@ -323,11 +328,11 @@ config.availableWrappers = {
title
:
'
Arhiivi lahtipakkija
'
,
description
:
''
,
id
:
'
uzip
'
,
port
:
process
.
env
.
SERVER_PORT
||
3007
,
port
:
3007
,
class
:
'
archiveExtractor
'
,
requestConf
:
{
requestBodyParamsMappings
:
{
isAsync
:
isAsyncParam
Value
isAsync
:
config
.
isAsyncParam
Description
},
requestFiles
:
{
content
:
{
...
...
@@ -350,12 +355,12 @@ config.availableWrappers = {
title
:
'
Sõnestaja pipe
'
,
description
:
''
,
id
:
'
tok
'
,
port
:
process
.
env
.
SERVER_PORT
||
3008
,
port
:
3008
,
class
:
'
inputOutputLocalCommand
'
,
command
:
config
.
availableCommands
.
TOKENIZER
,
requestConf
:
{
requestBodyParamsMappings
:
{
isAsync
:
isAsyncParam
Value
isAsync
:
config
.
isAsyncParam
Description
},
requestFiles
:
{
content
:
{
...
...
@@ -377,12 +382,12 @@ config.availableWrappers = {
CONCAT
:
{
title
:
'
Lihtne konkateneerija
'
,
id
:
'
concat
'
,
port
:
process
.
env
.
SERVER_PORT
||
3009
,
port
:
3009
,
class
:
'
simpleLocalCommand
'
,
command
:
config
.
availableCommands
.
CONCAT
,
requestConf
:
{
requestBodyParamsMappings
:
{
isAsync
:
isAsyncParam
Value
isAsync
:
config
.
isAsyncParam
Description
},
requestFiles
:
{
content
:
{
...
...
@@ -404,12 +409,12 @@ config.availableWrappers = {
MORPH_TAGGER
:
{
title
:
'
Morfoloogiline ühestaja pipe
'
,
id
:
'
tag
'
,
port
:
process
.
env
.
SERVER_PORT
||
3010
,
port
:
3010
,
class
:
'
inputOutputLocalCommand
'
,
command
:
config
.
availableCommands
.
MORPH_TAGGER
,
requestConf
:
{
requestBodyParamsMappings
:
{
isAsync
:
isAsyncParam
Value
isAsync
:
config
.
isAsyncParam
Description
},
requestFiles
:
{
content
:
{
...
...
src/model/serviceRequest.js
View file @
856f8d15
...
...
@@ -44,7 +44,7 @@ function ServiceRequest( requestBody, requestFiles ) {
continue
;
}
if
(
!
value
){
if
(
value
===
null
||
value
===
undefined
){
value
=
mapping
.
value
;
}
...
...
wrapper/inputOutputLocalCommand.js
View file @
856f8d15
...
...
@@ -13,41 +13,33 @@ function InputOutputLocalCommand(){
var
self
=
this
;
this
.
process
=
function
(
session
,
callback
)
{
var
contentFile
=
session
.
requestFiles
.
content
;
fs
.
readFile
(
contentFile
,
'
utf-8
'
,
function
(
err
,
sourceText
)
{
self
.
_getCommandModel
(
session
,
function
(
err
,
model
)
{
logger
.
debug
(
'
getCommandModel callback
'
);
if
(
err
)
return
callback
(
err
);
if
(
err
){
session
.
setErrors
(
err
);
return
callback
(
err
,
session
);
}
localExecutor
.
execute
(
model
,
session
,
function
(
err
,
response
)
{
self
.
_getCommandModel
(
session
,
function
(
err
,
model
)
{
logger
.
debug
(
'
getCommandModel callback
'
);
if
(
err
)
return
callback
(
err
);
localExecutor
.
execute
(
model
,
session
,
function
(
err
,
response
)
{
if
(
err
)
return
callback
(
err
);
logger
.
debug
(
'
Program is finished
'
);
if
(
response
.
isSuccess
){
session
.
message
=
Session
.
messages
.
OK
;
}
else
{
session
.
setErrors
(
response
.
errors
);
}
session
.
addOutputFile
(
'
output
'
,
{
type
:
'
output
'
,
fileName
:
config
.
wrapper
.
id
+
'
_output.
'
+
FileUtil
.
getExtension
(
model
.
outputPaths
.
output
),
filePath
:
model
.
outputPaths
.
output
,
contentType
:
mime
.
lookup
(
model
.
outputPaths
.
output
)
});
return
callback
(
err
,
session
);
logger
.
debug
(
'
Program is finished
'
);
if
(
response
.
isSuccess
){
session
.
message
=
Session
.
messages
.
OK
;
}
else
{
session
.
setErrors
(
response
.
errors
);
}
session
.
addOutputFile
(
'
output
'
,
{
type
:
'
output
'
,
fileName
:
config
.
wrapper
.
id
+
'
_output.
'
+
FileUtil
.
getExtension
(
model
.
outputPaths
.
output
),
filePath
:
model
.
outputPaths
.
output
,
contentType
:
mime
.
lookup
(
model
.
outputPaths
.
output
)
});
return
callback
(
err
,
session
);
});
});
};
this
.
_getCommandModel
=
function
(
session
,
callback
)
{
...
...
wrapper_dockers/README
deleted
100644 → 0
View file @
ceae0926
wrapper_dockers/lausestaja/Dockerfile
deleted
100644 → 0
View file @
ceae0926
# docker build -t kl_lausestaja .
#
# docker run --name kl_redis --rm --restart=no redis
#
# docker run --name kl_lausestaja --link kl_redis:redis -p 3001:3001 -v /wrapper/lausestaja/config:/config kl_lausestaja
#
# docker kill kl_lausestaja
#
# docker rm kl_lausestaja
#
#
FROM
ubuntu:14.04
ENV
TERM xterm
ENV
WRAPPER=LAUSESTAJA
ENV
SERVER_URL="http://dev.bitweb.ee"
ENV
SERVER_PORT=3001
RUN
apt-get update
&&
\
apt-get
-y
install
curl
&&
\
curl
-sL
https://deb.nodesource.com/setup |
sudo
bash -
&&
\
apt-get
-y
install
python build-essential nodejs
&&
\
apt-get
-y
install
git
&&
\
apt-get
-y
install
unzip
RUN
npm
install
-g
forever
RUN
mkdir
-p
/src
RUN
mkdir
-p
/tmp
RUN
mkdir
-p
/files
RUN
mkdir
-p
/logs
RUN
mkdir
-p
/config
RUN
cd
/src
&&
git clone
'https://github.com/BitWeb/keeleliin-wrapper-service.git'
.
&&
npm
install
&&
echo
"Run is Done"
RUN
mkdir
/utils
COPY
./utils /utils
RUN
unzip /utils/CGParser.zip
-d
/utils/
RUN
unzip /utils/morfyhestaja.zip
-d
/utils/
RUN
chmod
777
-Rf
/utils/
*
#Expose port
EXPOSE
$SERVER_PORT
VOLUME
["/tmp","/files","/logs", "/config"]
CMD
/./src/docker_start.sh
\ No newline at end of file
wrapper_dockers/lausestaja/utils/CGParser.zip
deleted
100644 → 0
View file @
ceae0926
File deleted
wrapper_dockers/lausestaja/utils/lausestaja.sh
deleted
100644 → 0
View file @
ceae0926
#!/bin/sh
BASEDIR
=
$(
dirname
$0
)
RADA
=
$BASEDIR
/morfyhestaja/konverter
cat
$1
|
$RADA
/rlausestaja.pl
wrapper_dockers/lausestaja/utils/morfyhestaja.zip
deleted
100644 → 0
View file @
ceae0926
File deleted
wrapper_dockers/morf_analysaator/Dockerfile
deleted