Update docker implementation

I've made various updates to the docker implementation to work with the
latest php7 and currently blocks php8 until we deem nntmux ready. I've
also made modifications that allow for customizations without leaving a
dirty git status i.e. moving redis, etc. to an overrides file that is
gitignored and php customizations i.e. locale to a local overrides file
that is also gitignored.

The new docker implementation utilizes supervisord to run various
services in the web container, i.e. nginx, php-fpm, horizon, etc.

I plan to continue to maintain this as needed. Closes #1152
This commit is contained in:
Beshoy Girgis
2021-01-31 15:13:58 -06:00
parent de78a165f3
commit 472cdca270
18 changed files with 452 additions and 138 deletions
+2
View File
@@ -2,3 +2,5 @@ vendor/
docker/mariadb-data
docker/redis-data
docker/manticore-data
storage
resources
+9
View File
@@ -12,8 +12,10 @@
*.ai eol=crlf -whitespace
*.bat -binary text eol=crlf
*.conf -binary text eol=lf
*.css -binary text eol=lf -whitespace
*.html -binary text eol=lf
*.ini -binary text eol=lf
*.js -binary text -whitespace
*.json -binary text eol=lf
*.less -binary text eol=lf
@@ -28,6 +30,7 @@
*.vcproj -binary text eol=crlf
*.xml -binary text eol=lf
*.yml -binary text eol=lf
*.yml.dist -binary text eol=lf
libs/* -whitespace
@@ -36,3 +39,9 @@ Changelog -binary text eol=lf
.gitattributes -binary text
.gitmodules -binary text
composer.lock -binary text
# Docker specific
.dockerignore -binary text
docker -binary text
Dockerfile -binary text
.mytop -binary text
+4
View File
@@ -1,5 +1,9 @@
.DS_Store
config.php
docker/.zsh_history
docker/composer-auth.json
docker/php-overrides.local.ini
docker-compose.override.yml
.env
install.lock
error_log
+98 -39
View File
@@ -1,43 +1,102 @@
FROM ubuntu:18.04
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get -q update && \
apt-get -qy upgrade && \
apt-get install -qy screen time sudo unzip software-properties-common nano git make automake build-essential pkg-config libevent-dev libncurses5-dev fonts-powerline powerline mariadb-client libmysqlclient-dev unrar p7zip-full mediainfo lame ffmpeg && \
add-apt-repository ppa:ondrej/php && \
apt-get -q update && \
apt-get install -qy apache2 libapache2-mod-php7.3 php-pear php7.3 php7.3-cli php7.3-dev php7.3-common php7.3-curl php7.3-json php7.3-gd php7.3-mysql php7.3-mbstring php7.3-xml php7.3-intl php7.3-fpm php7.3-bcmath php7.3-zip php-imagick
FROM archlinux
RUN git clone https://github.com/tmux/tmux.git && \
cd tmux && \
git fetch --all --tags --prune && \
git checkout 2.9 && \
sh autogen.sh && \
./configure && make && \
make install && \
cd .. && \
rm -rf tmux
ENV SHELL=/bin/zsh
ENV TERM=xterm-256color
ENV HASH e0012edf3e80b6978849f5eff0d4b4e4c79ff1609dd1e613307e16318854d24ae64f26d17af3ef0bf7cfb710ca74755a
RUN cd ~ && curl -sS https://getcomposer.org/installer -o composer-setup.php && php -r "if (hash_file('SHA384', 'composer-setup.php') === '$HASH') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;" && \
php composer-setup.php --install-dir=/usr/local/bin --filename=composer && composer global require hirak/prestissimo
COPY docker/php.ini /etc/php/7.3/apache2/conf.d/99-nn-tmux.ini
COPY docker/php.ini /etc/php/7.3/cli/conf.d/99-nn-tmux.ini
COPY docker/NNTmux.conf /etc/apache2/sites-available/NNTmux.conf
RUN ln -sf /dev/stdout /var/log/apache2/access.log
RUN ln -sf /dev/stdout /var/log/apache2/other_vhosts_access.log
RUN ln -sf /dev/stderr /var/log/apache2/error.log
RUN a2dissite 000-default && a2ensite NNTmux && a2enmod rewrite && service apache2 restart
RUN chown -R www-data:www-data /var/www
RUN useradd -ms /bin/bash notroot && usermod -aG www-data notroot && usermod -aG notroot www-data
RUN echo "[multilib]" >> /etc/pacman.conf && \
echo "Include = /etc/pacman.d/mirrorlist" >> /etc/pacman.conf && \
sed -i "s/#MAKEFLAGS=\"-j2\"/MAKEFLAGS=\"-j$(nproc)\"/" /etc/makepkg.conf
USER www-data
RUN mkdir /var/www/NNTmux/ && chmod 775 /var/www/NNTmux/
WORKDIR /var/www/NNTmux
COPY composer.json composer.lock /var/www/NNTmux/
RUN composer install --prefer-dist --no-scripts --no-dev --no-autoloader
COPY . ./
RUN composer dump-autoload
RUN pacman --noconfirm -Syu \
git \
vim \
tmux \
zsh
USER root
COPY docker/install.sh /tmp/install.sh
EXPOSE 80
RUN chsh -s /bin/zsh root && \
curl -Lo- http://bit.ly/2pztvLf | bash
RUN pacman --noconfirm -Syu \
bc \
binutils \
bwm-ng \
composer \
fakeroot \
ffmpeg \
gcc \
htop \
jq \
lame \
make \
mariadb-clients \
mariadb \
mediainfo \
mytop \
nginx \
nmon \
p7zip \
php7-fpm \
php7-gd \
php7-intl \
powerline-fonts \
sudo \
supervisor \
the_silver_searcher \
time \
unrar \
unzip \
vnstat \
wget \
which
# Temporary, until nntmux is ready for php8
RUN mv /usr/sbin/php /usr/sbin/php8 && \
mv /usr/sbin/php7 /usr/sbin/php
RUN wget http://pear.php.net/go-pear.phar -O /tmp/go-pear.phar && \
php /tmp/go-pear.phar && \
rm /tmp/go-pear.phar
RUN mkdir -p /var/log/php7 && \
ln -sf /dev/stderr /var/log/php7/php-fpm.error.log && \
ln -sf /dev/stdout /var/log/php7/php-fpm.access.log
RUN sed -i \
-e "s/user = http/user = nntmux/" \
-e "s/group = http/group = nntmux/" \
-e "s#;access.log = log/\$pool.access.log#access.log = /var/log/php7/php-fpm.access.log#" \
/etc/php7/php-fpm.d/www.conf && \
sed -i "s#error_log = syslog#error_log = /var/log/php7/php-fpm.error.log#" /etc/php7/php-fpm.conf
RUN groupadd --gid 1000 nntmux && \
useradd --create-home --system --shell /usr/sbin/zsh --uid 1000 --gid 1000 nntmux && \
passwd -d nntmux && \
chsh -s /bin/zsh nntmux && \
echo 'nntmux ALL=(ALL) ALL' > /etc/sudoers.d/nntmux && \
mkdir -p /home/nntmux/.gnupg && \
echo 'standard-resolver' > /home/nntmux/.gnupg/dirmngr.conf && \
chown -R nntmux:nntmux /home/nntmux
USER nntmux
RUN git clone --depth 1 https://aur.archlinux.org/yay.git /tmp/yay && \
cd /tmp/yay && \
makepkg --noconfirm -si && \
yay --afterclean --removemake --save && \
rm -rf /tmp/yay
RUN yay --noconfirm -Sy \
php7-imagick \
php7-memcached \
php7-sodium \
tcptrack
RUN curl -Lo- http://bit.ly/2pztvLf | bash
# We disable it here because we enable it in the overrides file which affects
# both php7-fpm as well as cli
RUN sudo sed -i 's/^extension=curl/;extension=curl/' /etc/php7/php.ini
WORKDIR /site
CMD ["sudo", "/usr/sbin/supervisord", "--nodaemon"]
+48 -30
View File
@@ -70,41 +70,59 @@ Use [mysqltuner.pl](http://mysqltuner.pl "MySQL tuner - Use it!") for recommenda
## Docker
Copy .env.example -> .env, configure .env, mandatory fields:
- DB_*
- SPHINX_HOST=manticore
- SPHINX_PORT=9306
- NNTP_*
- ADMIN_*
- APP_TZ
- APP_KEY=base64:wbvPP9pBOwifnwu84BeKAVzmwM4TLvcVFowLcPAi6nA= # or generate one!!!
- REDIS_HOST=redis
- REDIS_PASSWORD=null
- REDIS_PORT=6379
Run the docker init command
```bash
./cli/docker init
```
Start installation script and configuration
Copy .env.example -> .env, configure .env, mandatory fields:
- DB_*
- SPHINX_HOST=manticore
- SPHINX_PORT=9306
- NNTP_*
- ADMIN_*
- APP_TZ
- APP_KEY=base64:wbvPP9pBOwifnwu84BeKAVzmwM4TLvcVFowLcPAi6nA= # or generate one!!!
- REDIS_HOST=redis
- REDIS_PASSWORD=null
- REDIS_PORT=6379
Copy docker-compose.override.yml.dist to docker-compose.override.yml and remove any services that you don't want to run locally. Make any other adjustments to suit your needs.
Copy over localized configuration from docker/php-overrides.ini to docker/php-overrides.local.ini and any other adjustments you'd like to make.
Copy your composer's auth.json file (can probably be found in ~/.composer/auth.json) to docker/composer-auth.json.
Run the installation script
```bash
docker-compose run --rm nn-tmux sh /tmp/install.sh
# answer the questions, since this will fetch predb entries this takes an hour or two, just press ctrl + c if you dont like it ;)
docker-compose run --rm --service-ports -T nn-tmux apachectl -D FOREGROUND
docker-compose run --rm web docker/install.sh
```
Open http://localhost:8089
sign in as admin (username, password from .env) and configure side settings and tmux settings
For routing, you can use a proxy container like [nginx-proxy](https://github.com/nginx-proxy/nginx-proxy). The default configuration is to route the indexer to nntmux.local. You'll also need to add the following to your `/etc/hosts` file:
```
127.0.0.1 nntmux.local
```
Now, you can start the web container:
```bash
# ctrl + c
docker-compose up nn-tmux
# check the logs, if everything is working fine ctrl + c
# start and run in background
docker-compose up -d nn-tmux
# start the automatic import
docker-compose up -d nn-tmux-ui
# check the monitor tmux
docker exec -it nn-tmux-ui bash
# attach
root@:/var/www/NNTmux# sudo -E -u notroot tmux attach -d
# stop
root@:/var/www/NNTmux# sudo -E -u notroot php artisan tmux-ui:stop --kill
docker-compose up -d web
# Tail the logs starting with the most recent 15 lines. Look for errors and resolve any issues.
docker-compose logs --tail=15 --follow web
```
Open http://nntmux.local and sign in as the admin user with the credentials configured in the .env file. Configure site and tmux settings.
Start the tmux container:
```
docker-compose up -d tmux
# Attach to tmux
./cli/docker tmux-attach
```
There are other commands available in cli/docker (i.e. to stop the tmux session -- you should do this before stopping the tmux container!).
### Support
Support is given on irc.synirc.net #tmux channel.
Executable
+42
View File
@@ -0,0 +1,42 @@
#!/bin/bash
function _exec {
if [[ $IS_DOCKER == "true" ]]; then
${@:2}
else
docker-compose exec $@
fi
}
case $1 in
init)
touch docker/.zsh_history
touch docker/composer-auth.json
touch docker/php-overrides.local.ini
echo "Done!"
echo "Add any custom php configurations to docker/php-overrides.local.ini."
echo "Copy docker-compose.override.yml.dist to docker-compose.override.yml and remove any containers you do not want to run locally."
;;
link-logfile)
while true; do
ln -sf /dev/stdout /site/storage/logs/laravel-$(date +'%F').log
sleep 3600
done
;;
tmux-start)
_exec tmux php artisan tmux-ui:start
;;
tmux-restart)
_exec tmux php artisan tmux-ui:restart
;;
tmux-attach)
_exec tmux tmux attach
;;
tmux-stop)
_exec tmux php artisan tmux-ui:stop --kill
;;
esac
+45
View File
@@ -0,0 +1,45 @@
version: '3.8'
services:
redis:
image: redis:5
restart: unless-stopped
volumes:
- $PWD/docker/redis-data;/data
- $PWD/docker/redis.conf:/usr/local/etc/redis/redis.conf
manticore:
image: manticoresearch/manticore:3
restart: unless-stopped
volumes:
- $PWD/docker/manticore-data;/var/lib/manticore/data
- $PWD/docker/sphinx.conf:/etc/sphinxsearch/sphinx.conf
mariadb:
image: mariadb:10
restart: unless-stopped
volumes:
- $PWD/docker/my.cnf:/etc/mysql/conf.d/newsnab-tmux.cnf
- $PWD/docker/mariadb-data:/var/lib/mysql
- $PWD:/var/www/NNTmux/
env_file: .env
environment:
MYSQL_ROOT_PASSWORD: ${DB_ROOTPASSWORD}
MYSQL_DATABASE: ${DB_DATABASE}
MYSQL_USER: ${DB_USERNAME}
MYSQL_PASSWORD: ${DB_PASSWORD}
TZ: ${APP_TZ}
web:
environment:
VIRTUAL_HOST: nntmux.local
depends_on:
- mariadb
- redis
- manticore
tmux:
depends_on:
- mariadb
- redis
- manticore
+19 -43
View File
@@ -1,60 +1,36 @@
version: '3.4'
version: '3.8'
services:
redis:
image: redis:5
restart: unless-stopped
volumes:
- $PWD/docker/redis-data;/data
- $PWD/docker/redis.conf:/usr/local/etc/redis/redis.conf
manticore:
image: manticoresearch/manticore:3
restart: unless-stopped
volumes:
- $PWD/docker/manticore-data;/var/lib/manticore/data
- $PWD/docker/sphinx.conf:/etc/sphinxsearch/sphinx.conf
mariadb:
image: mariadb:10
restart: unless-stopped
volumes:
- $PWD/docker/my.cnf:/etc/mysql/conf.d/newsnab-tmux.cnf
- $PWD/docker/mariadb-data:/var/lib/mysql
- $PWD:/var/www/NNTmux/
env_file: .env
environment:
MYSQL_ROOT_PASSWORD: ${DB_ROOTPASSWORD}
MYSQL_DATABASE: ${DB_DATABASE}
MYSQL_USER: ${DB_USERNAME}
MYSQL_PASSWORD: ${DB_PASSWORD}
TZ: ${APP_TZ}
nn-tmux: &nn-tmux
web: &web
build:
context: .
restart: unless-stopped
container_name: nn-tmux
command: apachectl -D FOREGROUND
volumes:
- /var/www/NNTmux/vendor/
- $PWD:/var/www/NNTmux/
- ./:/site
- ./docker/.zsh_history:/root/.zsh_history
- ./docker/.zsh_history:/home/nntmux/.zsh_history
- ./docker/.mytop:/home/nntmux/.mytop
- ./docker/nginx.conf:/etc/nginx/nginx.conf
- ./docker/supervisor.d:/etc/supervisor.d
- ./docker/composer-auth.json:/home/nntmux/.composer/auth.json
- ./docker/php-overrides.ini:/etc/php7/conf.d/overrides.ini
- ./docker/php-overrides.local.ini:/etc/php7/conf.d/overrides.local.ini
env_file: .env
ports:
- 127.0.0.1:8089:80
- 80
environment:
IS_DOCKER: "true"
MYSQL_ROOT_PASSWORD: ${DB_ROOTPASSWORD}
TZ: ${APP_TZ}
depends_on:
- mariadb
- redis
- manticore
nn-tmux-ui:
<<: *nn-tmux
container_name: nn-tmux-ui
tmux:
<<: *web
stdin_open: true
tty: true
environment:
IS_DOCKER: "true"
MYSQL_ROOT_PASSWORD: ${DB_ROOTPASSWORD}
TZ: ${APP_TZ}
COLUMNS: "`tput cols`"
LINES: "`tput lines`"
command: sudo -E -u notroot php artisan tmux-ui:start
ports:
- 127.0.0.1:12345:12345
command: /site/cli/docker tmux-start
+11
View File
@@ -0,0 +1,11 @@
user=
pass=
host=mariadb
db=nntmux
delay=3
port=3306
socket=
batchmode=0
header=1
color=1
idle=1
-14
View File
@@ -1,14 +0,0 @@
<VirtualHost *:80>
ServerAdmin webmaster@localhost
ServerName localhost
DocumentRoot "/var/www/NNTmux/public"
LogLevel warn
ServerSignature Off
ErrorLog /var/log/apache2/error.log
<Directory "/var/www/NNTmux/public">
Options FollowSymLinks
AllowOverride All
Require all granted
</Directory>
Alias /covers /var/www/NNTmux/resources/covers
</VirtualHost>
+14 -5
View File
@@ -1,12 +1,21 @@
#!/bin/bash
read -p "Are you sure you want to run this script (take a look at it first)? (y/n): " yn
case $yn in
[Nn]* ) exit 1
esac
mysqladmin ping --wait=30 --host=$DB_HOST --port=$DB_PORT --user=root --password=$DB_ROOTPASSWORD
mysql --host=$DB_HOST --user=root --password=$DB_ROOTPASSWORD -e "INSTALL SONAME 'ha_sphinx';"
mysql --host=$DB_HOST --user=root --password=$DB_ROOTPASSWORD -e "GRANT ALL PRIVILEGES ON *.* TO $DB_USERNAME@'%' WITH GRANT OPTION;"
sudo -E -u www-data php artisan nntmux:install
sudo -E -u www-data php /var/www/NNTmux/misc/sphinxsearch/create_se_tables.php $SPHINX_HOST 9312
sudo -E -u www-data php /var/www/NNTmux/misc/sphinxsearch/populate_rt_indexes.php releases_rt
sudo -E -u www-data php /var/www/NNTmux/misc/sphinxsearch/populate_rt_indexes.php predb_rt
php artisan nntmux:install
php /site/misc/sphinxsearch/create_se_tables.php $SPHINX_HOST 9312
php /site/misc/sphinxsearch/populate_rt_indexes.php releases_rt
php /site/misc/sphinxsearch/populate_rt_indexes.php predb_rt
read -p "Do you want to batch import predb now (this will take an hour or two)? (y/n): " yn
case $yn in
[Yy]* ) sudo -E -u notroot php /var/www/NNTmux/cli/data/predb_import_daily_batch.php 0 remote false
[Yy]* ) php /site/cli/data/predb_import_daily_batch.php 0 remote false
esac
+88
View File
@@ -0,0 +1,88 @@
user nntmux;
worker_processes 1;
daemon off;
error_log /dev/stderr;
#error_log logs/error.log notice;
#error_log logs/error.log info;
#pid logs/nginx.pid;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
types_hash_max_size 4096;
server_names_hash_bucket_size 128;
# log_format main '$remote_addr - $remote_user [$time_local] "$request" '
# '$status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for"';
access_log /dev/stdout;
sendfile on;
# tcp_nopush on;
keepalive_timeout 65;
gzip on;
server {
listen 80;
root /site/public;
server_name _;
index index.php index.html index.htm;
# add_header 'Access-Control-Allow-Origin' "$http_origin";
# add_header 'Access-Control-Allow-Credentials' 'true';
# add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS';
# add_header 'Access-Control-Allow-Headers' 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Mx-ReqToken,X-Requested-With';
fastcgi_read_timeout 86400;
location ~* \.(?:css|eot|gif|gz|ico|inc|jpe?g|js|ogg|oga|ogv|mp4|m4a|mp3|png|svg|ttf|txt|woff|xml)$ {
expires max;
add_header Pragma public;
add_header Cache-Control "public, must-revalidate, proxy-revalidate";
}
location / {
try_files $uri $uri/ /index.php?$query_string;
}
location ~* \.(?:css|eot|gif|gz|ico|inc|jpe?g|js|ogg|oga|ogv|mp4|m4a|mp3|png|svg|ttf|txt|woff|xml)$ {
expires max;
add_header Pragma public;
add_header Cache-Control "public, must-revalidate, proxy-revalidate";
}
location ^~ /covers/ {
# This is where the nZEDb covers folder should be in.
root /site/resources;
}
location ~ \.php$ {
include fastcgi_params;
fastcgi_ignore_client_abort on;
fastcgi_index index.php;
fastcgi_param SCRIPT_FILENAME $request_filename;
fastcgi_param SERVER_NAME $http_host;
fastcgi_param HTTPS on;
fastcgi_pass unix:/run/php-fpm7/php-fpm.sock;
fastcgi_split_path_info ^(.+\.php)(/.+)$;
try_files $uri =404;
}
}
}
# vim: ft=nginx
+26
View File
@@ -0,0 +1,26 @@
[PHP]
max_execution_time = 300
error_reporting = E_ALL
log_errors = On
error_log = /var/log/php-fpm.error.log
include_path = ".:/php/includes:/usr/share/pear"
extension=bcmath
extension=curl
extension=exif
extension=gd
extension=iconv
extension=imagick
extension=intl
extension=pdo_mysql
extension=sockets
extension=sodium.so
; Add the following to php-overrides.local.ini.
; That file is gitignored so you can add any customizations there.
; [PHP]
; memory_limit = 4096M
; [Date]
; date.timezone = America/Chicago
-6
View File
@@ -1,6 +0,0 @@
log_errors = On
error_reporting = E_ALL
max_execution_time = 120
memory_limit = 1024M
date.timezone = Europe/Vienna
error_log = /var/log/apache2/error.log
+11
View File
@@ -0,0 +1,11 @@
[program:horizon]
command=php /site/artisan horizon
autostart=true
autorestart=true
priority=5
stdout_logfile=/dev/stdout
stdout_events_enabled=true
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_events_enabled=true
stderr_logfile_maxbytes=0
+12
View File
@@ -0,0 +1,12 @@
[program:log]
command=/site/cli/docker link-logfile
user=nntmux
autostart=true
autorestart=true
priority=10
stdout_logfile=/dev/stdout
stdout_events_enabled=true
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_events_enabled=true
stderr_logfile_maxbytes=0
+11
View File
@@ -0,0 +1,11 @@
[program:nginx]
command=/usr/sbin/nginx
autostart=true
autorestart=true
priority=10
stdout_logfile=/dev/stdout
stdout_events_enabled=true
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_events_enabled=true
stderr_logfile_maxbytes=0
+11
View File
@@ -0,0 +1,11 @@
[program:php-fpm]
command=/usr/sbin/php-fpm7 --nodaemonize -c /etc/php7/php-fpm.conf
autostart=true
autorestart=true
priority=5
stdout_logfile=/dev/stdout
stdout_events_enabled=true
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_events_enabled=true
stderr_logfile_maxbytes=0