зеркало из https://github.com/microsoft/pai.git
Add permission and new API in log-manager (#5046)
Add permission check in log-manage Add log-manager API to retrieve log
This commit is contained in:
Binyang2014
GitHub
Родитель
1e9580e472
Коммит
2ccdc1b958
|
|
@ -238,6 +238,10 @@ authentication:
|
|||
# uncomment following section if you want to customize the port of log-manager
|
||||
# log-manager:
|
||||
# port: 9103
|
||||
# admin_name: "admin"
|
||||
# admin_password: "admin"
|
||||
# jwt_secret: "jwt_secret"
|
||||
# token_expired_second: 120
|
||||
|
||||
|
||||
# uncomment following section if you want to customize the port of storage-manager
|
||||
|
|
|
|||
|
|
@ -128,6 +128,10 @@ rest-server:
|
|||
# uncomment following section if you want to customize the port of log-manager
|
||||
# log-manager:
|
||||
# port: 9103
|
||||
# admin_name: "admin"
|
||||
# admin_password: "admin"
|
||||
# jwt_secret: "jwt_secret"
|
||||
# token_expired_second: 120
|
||||
|
||||
# uncomment following section if you want to customize the port of storage-manager
|
||||
# storage-manager:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
# Copyright (c) Microsoft Corporation
|
||||
# All rights reserved.
|
||||
#
|
||||
# MIT License
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
|
||||
# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
|
||||
# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
|
||||
# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
|
||||
# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
FROM alpine:3.10
|
||||
|
||||
# install dev tools
|
||||
RUN apk update && apk add --no-cache tini bash findutils
|
||||
COPY src/cleaner/ /usr/bin/cleaner/
|
||||
ENTRYPOINT ["/sbin/tini","--","/usr/bin/cleaner/entrypoint.sh"]
|
||||
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
# Original work Copyright (c) 2015 Steffen Bleul
|
||||
# Modified work Copyright (c) Microsoft Corporation
|
||||
# All rights reserved.
|
||||
#
|
||||
# MIT License
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
|
||||
# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
|
||||
# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
|
||||
# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
|
||||
# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
FROM alpine:3.10
|
||||
|
||||
# logrotate version (e.g. 3.9.1-r0)
|
||||
ARG LOGROTATE_VERSION=latest
|
||||
# permissions
|
||||
ARG CONTAINER_UID=1000
|
||||
ARG CONTAINER_GID=1000
|
||||
|
||||
# install dev tools
|
||||
RUN export CONTAINER_USER=logrotate && \
|
||||
export CONTAINER_GROUP=logrotate && \
|
||||
addgroup -g $CONTAINER_GID logrotate && \
|
||||
adduser -u $CONTAINER_UID -G logrotate -h /usr/bin/logrotate.d -s /bin/bash -S logrotate && \
|
||||
apk add --update \
|
||||
tini \
|
||||
bash \
|
||||
tar \
|
||||
gzip \
|
||||
wget \
|
||||
tzdata && \
|
||||
if [ "${LOGROTATE_VERSION}" = "latest" ]; \
|
||||
then apk add logrotate ; \
|
||||
else apk add "logrotate=${LOGROTATE_VERSION}" ; \
|
||||
fi && \
|
||||
mkdir -p /usr/bin/logrotate.d && \
|
||||
wget --no-check-certificate -O /tmp/go-cron.tar.gz https://github.com/michaloo/go-cron/releases/download/v0.0.2/go-cron.tar.gz && \
|
||||
tar xvf /tmp/go-cron.tar.gz -C /usr/bin && \
|
||||
apk del \
|
||||
wget && \
|
||||
rm -rf /var/cache/apk/* && rm -rf /tmp/*
|
||||
|
||||
COPY src/logrotate/ /usr/bin/logrotate.d/
|
||||
RUN chmod +x /usr/bin/logrotate.d/docker-entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/sbin/tini","--","/usr/bin/logrotate.d/docker-entrypoint.sh"]
|
||||
VOLUME ["/logrotate-status"]
|
||||
CMD ["cron"]
|
||||
|
|
@ -15,5 +15,11 @@
|
|||
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
FROM openresty/openresty:1.15.8.2-alpine
|
||||
COPY src/nginx/nginx.conf /etc/nginx/conf.d/default.conf
|
||||
FROM openresty/openresty:1.15.8.3-2-alpine-fat
|
||||
|
||||
RUN luarocks install lua-cjson && luarocks install lua-resty-jwt && \
|
||||
luarocks install luafilesystem
|
||||
|
||||
COPY src/nginx/nginx.conf.default /etc/nginx/conf.d/default.conf
|
||||
COPY src/nginx/nginx.conf /usr/local/openresty/nginx/conf/nginx.conf
|
||||
COPY src/nginx/*.lua /etc/nginx/lua/
|
||||
|
|
|
|||
|
|
@ -1,46 +1,43 @@
|
|||
## Log-manager section parser
|
||||
|
||||
- [Default Configuration](#D_Config)
|
||||
- [How to Configure](#HT_Config)
|
||||
- [Generated Configuration](#G_Config)
|
||||
- [Data Table](#T_config)
|
||||
- [Default configuration](#default-configuration)
|
||||
- [How to configure cluster section in service-configuration.yaml](#how-to-configure-cluster-section-in-service-configurationyaml)
|
||||
- [Generated Configuration](#generated-configuration)
|
||||
- [Table](#table)
|
||||
|
||||
#### Default configuration <a name="D_Config"></a>
|
||||
#### Default configuration
|
||||
|
||||
[log-manager default configuration](log-manager.yaml)
|
||||
|
||||
#### How to configure cluster section in service-configuration.yaml <a name="HT_Config"></a>
|
||||
#### How to configure cluster section in service-configuration.yaml
|
||||
|
||||
All configurations in this section is optional. If you want to customized these value, you can configure it in service-configuration.yaml.
|
||||
|
||||
For example, if you want to use different port than the default 9103, add following to your service-configuration.yaml as following:
|
||||
```yaml
|
||||
log-manager:
|
||||
port: new-value
|
||||
port: new-value
|
||||
```
|
||||
|
||||
#### Generated Configuration <a name="G_Config"></a>
|
||||
#### Generated Configuration
|
||||
|
||||
Generated configuration means the object model after parsing. The parsed data will be presented by a yaml format.
|
||||
```yaml
|
||||
log-manager:
|
||||
port: 9103
|
||||
port: 9103
|
||||
admin_name: admin
|
||||
admin_password: admin
|
||||
jwt_secret: "jwt_secret"
|
||||
token_expired_second: 120
|
||||
```
|
||||
|
||||
|
||||
#### Table <a name="T_Config"></a>
|
||||
#### Table
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td>Data in Configuration File</td>
|
||||
<td>Data in Cluster Object Model</td>
|
||||
<td>Data in Jinja2 Template</td>
|
||||
<td>Data type</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>log-manager.port</td>
|
||||
<td>com["log-manager"]["port"]</td>
|
||||
<td>cluster_cfg["log-manager"]["port"]</td>
|
||||
<td>Int</td>
|
||||
</tr>
|
||||
</table>
|
||||
| Data in Configuration File | Data in Cluster Object Model | Data in Jinja2 Template | Data type |
|
||||
|-----------------------------------|---------------------------------------------|----------------------------------------------------|-----------|
|
||||
| log-manager.port | com["log-manager"]["port"] | cluster_cfg["log-manager"]["port"] | Int |
|
||||
| log-manager.admin_name | com["log-manager"]["admin_name"] | cluster_cfg["log-manager"]["admin_name"] | String |
|
||||
| log-manager.admin_password | com["log-manager"]["admin_password"] | cluster_cfg["log-manager"]["admin_password"] | String |
|
||||
| log-manager.jwt_secret | com["log-manager"]["jwt_secret"] | cluster_cfg["log-manager"]["jwt_secret"] | String |
|
||||
| log-manager.token_expired_second | com["log-manager"]["token_expired_second"] | cluster_cfg["log-manager"]["token_expired_second"] | Int |
|
||||
|
|
|
|||
|
|
@ -18,3 +18,7 @@
|
|||
service_type: "k8s"
|
||||
|
||||
port: 9103
|
||||
admin_name: "admin"
|
||||
admin_password: "admin"
|
||||
jwt_secret: "jwt_secret"
|
||||
token_expired_second: 120
|
||||
|
|
|
|||
|
|
@ -34,12 +34,9 @@ spec:
|
|||
priorityClassName: pai-daemon-priority
|
||||
hostNetwork: false
|
||||
containers:
|
||||
- name: log-manager-logrotate
|
||||
image: {{ cluster_cfg["cluster"]["docker-registry"]["prefix"] }}log-manager-logrotate:{{ cluster_cfg["cluster"]["docker-registry"]["tag"] }}
|
||||
- name: log-cleaner
|
||||
image: {{ cluster_cfg["cluster"]["docker-registry"]["prefix"] }}log-manager-cleaner:{{ cluster_cfg["cluster"]["docker-registry"]["tag"] }}
|
||||
imagePullPolicy: Always
|
||||
env:
|
||||
- name: LOGROTATE_CRONSCHEDULE
|
||||
value: "*/10 * * * *"
|
||||
volumeMounts:
|
||||
- name: pai-log
|
||||
mountPath: /usr/local/pai/logs
|
||||
|
|
@ -77,6 +74,19 @@ spec:
|
|||
cpu: 0
|
||||
memory: "128Mi"
|
||||
{%- endif %}
|
||||
env:
|
||||
- name: ADMIN_NAME
|
||||
value: {{ cluster_cfg["log-manager"]["admin_name"] }}
|
||||
- name: ADMIN_PASSWORD
|
||||
value: {{ cluster_cfg["log-manager"]["admin_password"] }}
|
||||
- name: JWT_SECRET
|
||||
value: {{ cluster_cfg["log-manager"]["jwt_secret"] }}
|
||||
- name: TOKEN_EXPIRED_SECOND
|
||||
value: '{{ cluster_cfg["log-manager"]["token_expired_second"] }}'
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
volumes:
|
||||
- name: pai-log
|
||||
hostPath:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,43 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright (c) Microsoft Corporation
|
||||
# All rights reserved.
|
||||
#
|
||||
# MIT License
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
|
||||
# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
|
||||
# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
|
||||
# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
|
||||
# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
|
||||
log_exist_time=30 # 30 day
|
||||
if [ -n "${LOG_EXIST_TIME}" ]; then
|
||||
log_exist_time=${LOG_EXIST_TIME}
|
||||
fi
|
||||
|
||||
cat > /etc/periodic/daily/remove_logs << EOF
|
||||
#!/bin/bash
|
||||
/usr/bin/pgrep -f ^find 2>&1 > /dev/null || find /usr/local/pai/logs/* -mtime +${log_exist_time} -type f -exec rm -fv {} \;
|
||||
EOF
|
||||
|
||||
cat > /etc/periodic/weekly/remove_log_dir << EOF
|
||||
#!/bin/bash
|
||||
"/usr/bin/pgrep -f ^find 2>&1 > /dev/null || find /usr/local/pai/logs/* -mtime +${log_exist_time} -type d -empty -exec rmdir -v {} \;"
|
||||
EOF
|
||||
|
||||
chmod a+x /etc/periodic/daily/remove_logs /etc/periodic/weekly/remove_log_dir
|
||||
|
||||
echo "cron job added"
|
||||
|
||||
crond -f -l 0
|
||||
|
||||
|
|
@ -1,97 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Original work Copyright (c) 2015 Steffen Bleul
|
||||
# Modified work Copyright (c) Microsoft Corporation
|
||||
# All rights reserved.
|
||||
#
|
||||
# MIT License
|
||||
|
||||
|
||||
# A helper script for ENTRYPOINT.
|
||||
|
||||
set -e
|
||||
|
||||
[[ ${DEBUG} == true ]] && set -x
|
||||
|
||||
if [ -n "${DELAYED_START}" ]; then
|
||||
sleep ${DELAYED_START}
|
||||
fi
|
||||
|
||||
# Logrotate status file handling
|
||||
readonly logrotate_logstatus=${LOGROTATE_STATUSFILE:-"/logrotate-status/logrotate.status"}
|
||||
|
||||
# ----- Crontab Generation ------
|
||||
|
||||
logrotate_parameters=""
|
||||
|
||||
if [ -n "${LOGROTATE_PARAMETERS}" ]; then
|
||||
logrotate_parameters="-"${LOGROTATE_PARAMETERS}
|
||||
fi
|
||||
|
||||
syslogger_tag=""
|
||||
|
||||
if [ -n "${SYSLOGGER_TAG}" ]; then
|
||||
syslogger_tag=" -t "${SYSLOGGER_TAG}
|
||||
fi
|
||||
|
||||
syslogger_command=""
|
||||
|
||||
if [ -n "${SYSLOGGER}" ]; then
|
||||
syslogger_command="logger "${syslogger_tag}
|
||||
fi
|
||||
|
||||
logrotate_cronlog=""
|
||||
|
||||
if [ -n "${LOGROTATE_LOGFILE}" ] && [ -z "${SYSLOGGER}"]; then
|
||||
logrotate_cronlog=" 2>&1 | tee -a "${LOGROTATE_LOGFILE}
|
||||
else
|
||||
if [ -n "${SYSLOGGER}" ]; then
|
||||
logrotate_cronlog=" 2>&1 | "${syslogger_command}
|
||||
fi
|
||||
fi
|
||||
|
||||
logrotate_croninterval="1 0 0 * * *"
|
||||
|
||||
if [ -n "${LOGROTATE_INTERVAL}" ]; then
|
||||
case "$LOGROTATE_INTERVAL" in
|
||||
hourly)
|
||||
logrotate_croninterval='@hourly'
|
||||
;;
|
||||
daily)
|
||||
logrotate_croninterval='@daily'
|
||||
;;
|
||||
weekly)
|
||||
logrotate_croninterval='@weekly'
|
||||
;;
|
||||
monthly)
|
||||
logrotate_croninterval='@monthly'
|
||||
;;
|
||||
yearly)
|
||||
logrotate_croninterval='@yearly'
|
||||
;;
|
||||
*)
|
||||
logrotate_croninterval="1 0 0 * * *"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
if [ -n "${LOGROTATE_CRONSCHEDULE}" ]; then
|
||||
logrotate_croninterval=${LOGROTATE_CRONSCHEDULE}
|
||||
fi
|
||||
|
||||
logrotate_cron_timetable="/usr/bin/pgrep -f ^/usr/sbin/logrotate 2>&1 > /dev/null || /usr/sbin/logrotate ${logrotate_parameters} --state=${logrotate_logstatus} /usr/bin/logrotate.d/logrotate.conf ${logrotate_cronlog}"
|
||||
|
||||
log_exist_time=30 # 30 day
|
||||
if [ -n "${LOG_EXIST_TIME}" ]; then
|
||||
log_exist_time=${LOG_EXIST_TIME}
|
||||
fi
|
||||
|
||||
# ----- Cron Start ------
|
||||
exec /usr/bin/go-cron '@daily' /bin/bash -c "/usr/bin/pgrep -f ^find 2>&1 > /dev/null || find /usr/local/pai/logs/* -mtime +${log_exist_time} -type f -exec rm -fv {} \;"&
|
||||
if [ "$1" = 'cron' ]; then
|
||||
exec /usr/bin/go-cron "${logrotate_croninterval}" /bin/bash -c "${logrotate_cron_timetable}"
|
||||
fi
|
||||
|
||||
#-----------------------
|
||||
|
||||
exec "$@"
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
/usr/local/pai/logs/*/*/*/*/*.log
|
||||
/usr/local/pai/logs/*/*/*/*/*.stdout
|
||||
/usr/local/pai/logs/*/*/*/*/*.stderr
|
||||
/usr/local/pai/logs/*/*/*/*/*.all
|
||||
{
|
||||
nomail
|
||||
rotate 1
|
||||
nocompress
|
||||
missingok
|
||||
notifempty
|
||||
copytruncate
|
||||
size 256M
|
||||
maxage 30
|
||||
}
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
-- Copyright (c) Microsoft Corporation
|
||||
-- All rights reserved.
|
||||
-- MIT License
|
||||
-- Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
|
||||
-- documentation files (the "Software"), to deal in the Software without restriction, including without limitation
|
||||
-- the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
|
||||
-- to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
-- The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
-- THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
|
||||
-- BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
-- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
-- DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
|
||||
local lfs = require "lfs"
|
||||
|
||||
local function get_rotated_log(log_path)
|
||||
for file in lfs.dir(log_path) do
|
||||
local rotated_log_name = string.match(file, "^@.*%.s")
|
||||
if rotated_log_name then
|
||||
return rotated_log_name
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
local args = ngx.req.get_uri_args()
|
||||
local username = args["username"]
|
||||
local framework_name = args["framework-name"]
|
||||
local taskrole = args["taskrole"]
|
||||
local pod_uid = args["pod-uid"]
|
||||
local token = args["token"]
|
||||
local tail_mode = args["tail-mode"]
|
||||
|
||||
if not token or not username or not taskrole or not framework_name or not pod_uid then
|
||||
ngx.log(ngx.ERR, "some query parameters is nil")
|
||||
ngx.status = ngx.HTTP_BAD_REQUEST
|
||||
return ngx.exit(ngx.HTTP_OK)
|
||||
end
|
||||
|
||||
local path_prefix = "/usr/local/pai/logs/"..username.."/".. framework_name.."/".. taskrole.."/"..pod_uid.."/"
|
||||
local log_name = ngx.var[1]
|
||||
|
||||
local log_path = path_prefix..log_name
|
||||
ngx.log(ngx.INFO, "get log name "..log_name)
|
||||
if string.match(log_name, "^user%-.*$") then
|
||||
-- we only keep one rotated log in log manager
|
||||
if string.match(log_name, "%.1$") then
|
||||
local parent_path = path_prefix..string.sub(log_name, 1, string.len(log_name) - 2)
|
||||
local rotated_log_name = get_rotated_log(parent_path)
|
||||
if not rotated_log_name then
|
||||
ngx.status = ngx.HTTP_NOT_FOUND
|
||||
return ngx.exit(ngx.HTTP_OK)
|
||||
else
|
||||
log_path = parent_path.."/"..rotated_log_name
|
||||
end
|
||||
else
|
||||
log_path = path_prefix..log_name.."/current"
|
||||
end
|
||||
end
|
||||
|
||||
ngx.log(ngx.INFO, "get log from path"..log_path)
|
||||
|
||||
if lfs.attributes(log_path, "mode") ~= "file" then
|
||||
ngx.log(ngx.ERR, log_path.." not exists")
|
||||
ngx.status = ngx.HTTP_NOT_FOUND
|
||||
return ngx.exit(ngx.HTTP_OK)
|
||||
end
|
||||
|
||||
local logs
|
||||
if (tail_mode == "true") then
|
||||
logs = io.popen("tail -c 16k "..log_path)
|
||||
else
|
||||
logs = io.popen("cat "..log_path)
|
||||
end
|
||||
|
||||
-- buffer size (8K)
|
||||
local size = 2^13
|
||||
while true do
|
||||
local block = logs:read(size)
|
||||
if not block then break end
|
||||
ngx.say(block)
|
||||
end
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
-- Copyright (c) Microsoft Corporation
|
||||
-- All rights reserved.
|
||||
-- MIT License
|
||||
-- Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
|
||||
-- documentation files (the "Software"), to deal in the Software without restriction, including without limitation
|
||||
-- the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
|
||||
-- to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
-- The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
-- THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
|
||||
-- BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
-- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
-- DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
local jwt = require "resty.jwt"
|
||||
local validators = require "resty.jwt-validators"
|
||||
|
||||
local args = ngx.req.get_uri_args()
|
||||
local jwt_token = args["token"]
|
||||
if not jwt_token then
|
||||
ngx.status = ngx.HTTP_FORBIDDEN
|
||||
return ngx.exit(ngx.HTTP_OK)
|
||||
end
|
||||
|
||||
local jwt_secret = os.getenv("JWT_SECRET")
|
||||
local node_name = os.getenv("NODE_NAME")
|
||||
|
||||
local claim_spec = {
|
||||
sub = validators.equals("log-manager-"..node_name),
|
||||
exp = validators.is_not_expired()
|
||||
}
|
||||
local jwt_obj = jwt:verify(jwt_secret, jwt_token, claim_spec)
|
||||
|
||||
if not jwt_obj["verified"] then
|
||||
ngx.status = ngx.HTTP_FORBIDDEN
|
||||
ngx.header["Access-Control-Allow-Origin"] = "*";
|
||||
return ngx.exit(ngx.HTTP_OK)
|
||||
end
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
-- Copyright (c) Microsoft Corporation
|
||||
-- All rights reserved.
|
||||
-- MIT License
|
||||
-- Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
|
||||
-- documentation files (the "Software"), to deal in the Software without restriction, including without limitation
|
||||
-- the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
|
||||
-- to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
-- The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
-- THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
|
||||
-- BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
-- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
-- DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
local cjson = require "cjson"
|
||||
local lfs = require "lfs"
|
||||
|
||||
local function has_file_with_pattern(path, pattern)
|
||||
for file in lfs.dir(path) do
|
||||
if string.match(file, pattern) then
|
||||
return true
|
||||
end
|
||||
end
|
||||
return false
|
||||
end
|
||||
|
||||
local function is_dir(path)
|
||||
return lfs.attributes(path, "mode") == "directory"
|
||||
end
|
||||
|
||||
local args = ngx.req.get_uri_args()
|
||||
local username = args["username"]
|
||||
local framework_name = args["framework-name"]
|
||||
local taskrole = args["taskrole"]
|
||||
local pod_uid = args["pod-uid"]
|
||||
local token = args["token"]
|
||||
|
||||
if not token or not username or not taskrole or not framework_name or not pod_uid then
|
||||
ngx.log(ngx.ERR, "some query parameters is nil")
|
||||
ngx.status = ngx.HTTP_BAD_REQUEST
|
||||
return ngx.exit(ngx.HTTP_OK)
|
||||
end
|
||||
|
||||
local log_query_param = "?username="..username.."&framework-name="..framework_name..
|
||||
"&pod-uid="..pod_uid.."&taskrole="..taskrole.."&token="..token
|
||||
local path = "/usr/local/pai/logs/"..username.."/".. framework_name.."/".. taskrole.."/"..pod_uid.."/"
|
||||
local path_prefix = "/api/v1/logs/"
|
||||
|
||||
local ret = {}
|
||||
|
||||
if not is_dir(path) then
|
||||
ngx.log(ngx.ERR, "log folder not exists")
|
||||
ngx.status = ngx.HTTP_NOT_FOUND
|
||||
return ngx.exit(ngx.HTTP_OK)
|
||||
end
|
||||
|
||||
for file in lfs.dir(path) do
|
||||
if not is_dir(path..file) then
|
||||
if string.match(file, "^user%.pai%..*$") then
|
||||
local sub_str = string.sub(file, string.len("user.pai.") + 1)
|
||||
ret[sub_str] = path_prefix..file..log_query_param
|
||||
else
|
||||
ret[file] = path_prefix..file..log_query_param
|
||||
end
|
||||
elseif string.match(file, "^user-.*$") then
|
||||
local sub_str = string.sub(file, string.len("user-") + 1)
|
||||
ret[sub_str] = path_prefix..file..log_query_param
|
||||
if has_file_with_pattern(path..file, "^@.*%.s") then
|
||||
ret[sub_str..".1"] = path_prefix..file..".1"..log_query_param
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
ngx.say(cjson.encode(ret))
|
||||
|
|
@ -15,70 +15,74 @@
|
|||
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name localhost;
|
||||
client_max_body_size 0; # Disable checking of client request body size.
|
||||
client_body_buffer_size 256M;
|
||||
proxy_connect_timeout 60m;
|
||||
proxy_send_timeout 60m;
|
||||
proxy_read_timeout 60m;
|
||||
send_timeout 60m;
|
||||
|
||||
#
|
||||
# Health check
|
||||
#
|
||||
location = /healthz {
|
||||
default_type text/plain;
|
||||
return 200 "Log manager ready.";
|
||||
}
|
||||
# nginx.conf -- docker-openresty
|
||||
#
|
||||
# This file is installed to:
|
||||
# `/usr/local/openresty/nginx/conf/nginx.conf`
|
||||
# and is the file loaded by nginx at startup,
|
||||
# unless the user specifies otherwise.
|
||||
#
|
||||
# It tracks the upstream OpenResty's `nginx.conf`, but removes the `server`
|
||||
# section and adds this directive:
|
||||
# `include /etc/nginx/conf.d/*.conf;`
|
||||
#
|
||||
# The `docker-openresty` file `nginx.vh.default.conf` is copied to
|
||||
# `/etc/nginx/conf.d/default.conf`. It contains the `server section
|
||||
# of the upstream `nginx.conf`.
|
||||
#
|
||||
# See https://github.com/openresty/docker-openresty/blob/master/README.md#nginx-config-files
|
||||
#
|
||||
|
||||
#
|
||||
# Get all logs
|
||||
#
|
||||
location /log-manager {
|
||||
add_header Access-Control-Allow-Origin *;
|
||||
add_header Access-Control-Allow-Methods 'GET, POST, OPTIONS';
|
||||
default_type text/plain;
|
||||
alias /usr/local/pai/logs;
|
||||
autoindex on;
|
||||
autoindex_exact_size off;
|
||||
autoindex_localtime on;
|
||||
}
|
||||
#user nobody;
|
||||
worker_processes 1;
|
||||
|
||||
#
|
||||
# Get full/tailed log
|
||||
#
|
||||
location ~ ^/log-manager/(full|tail)/(.*)$ {
|
||||
add_header Access-Control-Allow-Origin *;
|
||||
add_header Access-Control-Allow-Methods 'GET, POST, OPTIONS';
|
||||
default_type text/plain;
|
||||
content_by_lua_block{
|
||||
logpath = "/usr/local/pai/logs/"..ngx.var[2]
|
||||
errcheck = " || echo No such file!"
|
||||
if (ngx.var[1] == "tail")
|
||||
then
|
||||
logs = io.popen("tail -c 16k "..logpath..errcheck)
|
||||
elseif (ngx.var[1] == "full")
|
||||
then
|
||||
logs = io.popen("cat "..logpath..errcheck)
|
||||
end
|
||||
for line in logs:lines() do
|
||||
ngx.say(line)
|
||||
end
|
||||
}
|
||||
}
|
||||
#error_log logs/error.log;
|
||||
#error_log logs/error.log notice;
|
||||
error_log logs/error.log info;
|
||||
|
||||
#
|
||||
# Get compressed logs for old job
|
||||
#
|
||||
location /log-backup {
|
||||
add_header Access-Control-Allow-Origin *;
|
||||
add_header Access-Control-Allow-Methods 'GET, POST, OPTIONS';
|
||||
default_type text/plain;
|
||||
alias /usr/local/pai/logs-backup;
|
||||
autoindex on;
|
||||
autoindex_exact_size off;
|
||||
autoindex_localtime on;
|
||||
}
|
||||
#pid logs/nginx.pid;
|
||||
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
|
||||
http {
|
||||
include mime.types;
|
||||
default_type application/octet-stream;
|
||||
|
||||
#log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
# '$status $body_bytes_sent "$http_referer" '
|
||||
# '"$http_user_agent" "$http_x_forwarded_for"';
|
||||
|
||||
#access_log logs/access.log main;
|
||||
|
||||
# See Move default writable paths to a dedicated directory (#119)
|
||||
# https://github.com/openresty/docker-openresty/issues/119
|
||||
client_body_temp_path /var/run/openresty/nginx-client-body;
|
||||
proxy_temp_path /var/run/openresty/nginx-proxy;
|
||||
fastcgi_temp_path /var/run/openresty/nginx-fastcgi;
|
||||
uwsgi_temp_path /var/run/openresty/nginx-uwsgi;
|
||||
scgi_temp_path /var/run/openresty/nginx-scgi;
|
||||
|
||||
sendfile on;
|
||||
#tcp_nopush on;
|
||||
|
||||
#keepalive_timeout 0;
|
||||
keepalive_timeout 65;
|
||||
|
||||
#gzip on;
|
||||
|
||||
init_by_lua_block {
|
||||
local errlog = require "ngx.errlog"
|
||||
errlog.set_filter_level(ngx.INFO)
|
||||
}
|
||||
include /etc/nginx/conf.d/*.conf;
|
||||
}
|
||||
env ADMIN_NAME;
|
||||
env ADMIN_PASSWORD;
|
||||
env JWT_SECRET;
|
||||
env TOKEN_EXPIRED_SECOND;
|
||||
env NODE_NAME;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,77 @@
|
|||
# Copyright (c) Microsoft Corporation
|
||||
# All rights reserved.
|
||||
#
|
||||
# MIT License
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
|
||||
# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
|
||||
# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
|
||||
# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
|
||||
# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name localhost;
|
||||
client_max_body_size 0; # Disable checking of client request body size.
|
||||
client_body_buffer_size 256M;
|
||||
proxy_connect_timeout 60m;
|
||||
proxy_send_timeout 60m;
|
||||
proxy_read_timeout 60m;
|
||||
send_timeout 60m;
|
||||
|
||||
#
|
||||
# Health check
|
||||
#
|
||||
location = /healthz {
|
||||
default_type text/plain;
|
||||
return 200 "Log manager ready.";
|
||||
}
|
||||
|
||||
#
|
||||
# Get log list
|
||||
#
|
||||
location /api/v1/logs {
|
||||
add_header Access-Control-Allow-Origin *;
|
||||
add_header Access-Control-Allow-Methods 'GET';
|
||||
limit_except GET {
|
||||
deny all;
|
||||
}
|
||||
default_type application/json;
|
||||
access_by_lua_file /etc/nginx/lua/guard.lua;
|
||||
content_by_lua_file /etc/nginx/lua/list_logs.lua;
|
||||
}
|
||||
|
||||
#
|
||||
# Get the token
|
||||
#
|
||||
location /api/v1/tokens {
|
||||
add_header Access-Control-Allow-Origin *;
|
||||
add_header Access-Control-Allow-Methods 'POST';
|
||||
limit_except POST {
|
||||
deny all;
|
||||
}
|
||||
default_type application/json;
|
||||
content_by_lua_file /etc/nginx/lua/token.lua;
|
||||
}
|
||||
|
||||
#
|
||||
# Get full/tail log
|
||||
#
|
||||
location ~ ^/api/v1/logs/(.*)$ {
|
||||
add_header Access-Control-Allow-Origin *;
|
||||
add_header Access-Control-Allow-Methods 'GET';
|
||||
limit_except GET {
|
||||
deny all;
|
||||
}
|
||||
default_type text/plain;
|
||||
access_by_lua_file /etc/nginx/lua/guard.lua;
|
||||
content_by_lua_file /etc/nginx/lua/get_log_content.lua;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
-- Copyright (c) Microsoft Corporation
|
||||
-- All rights reserved.
|
||||
-- MIT License
|
||||
-- Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
|
||||
-- documentation files (the "Software"), to deal in the Software without restriction, including without limitation
|
||||
-- the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
|
||||
-- to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
-- The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
-- THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
|
||||
-- BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
-- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
-- DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
local cjson = require "cjson"
|
||||
local jwt = require "resty.jwt"
|
||||
|
||||
-- check username & password
|
||||
local admin_name = os.getenv("ADMIN_NAME")
|
||||
local admin_password = os.getenv("ADMIN_PASSWORD")
|
||||
local token_expired_second = os.getenv("TOKEN_EXPIRED_SECOND")
|
||||
ngx.req.read_body()
|
||||
local ok, body = pcall(cjson.decode, ngx.req.get_body_data())
|
||||
|
||||
if not ok then
|
||||
ngx.status = ngx.HTTP_BAD_REQUEST
|
||||
return ngx.exit(ngx.HTTP_OK)
|
||||
end
|
||||
|
||||
if body["username"] ~= admin_name or body["password"] ~= admin_password then
|
||||
ngx.status = ngx.HTTP_UNAUTHORIZED
|
||||
return ngx.exit(ngx.HTTP_OK)
|
||||
end
|
||||
|
||||
-- sign jwt token
|
||||
local jwt_secret = os.getenv("JWT_SECRET")
|
||||
local node_name = os.getenv("NODE_NAME")
|
||||
local jwt_token = jwt:sign(
|
||||
jwt_secret,
|
||||
{
|
||||
header={typ="JWT", alg="HS256"},
|
||||
payload={sub="log-manager-"..node_name, iat=os.time(), exp=os.time() + tonumber(token_expired_second)}
|
||||
}
|
||||
)
|
||||
ngx.say(cjson.encode({token=jwt_token}))
|
||||
Загрузка…
Ссылка в новой задаче