support binding the http server to a unix socket file instead of TCP socket

This commit is contained in:
Daniel Pavel 2019-06-10 20:19:27 +03:00
parent f40fc5aa75
commit e254565901
8 changed files with 167 additions and 166 deletions

5
cps.py
View File

@ -28,8 +28,8 @@ sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'vendor
from cps import create_app from cps import create_app
from cps import web_server
from cps.opds import opds from cps.opds import opds
from cps import Server
from cps.web import web from cps.web import web
from cps.jinjia import jinjia from cps.jinjia import jinjia
from cps.about import about from cps.about import about
@ -56,7 +56,8 @@ def main():
app.register_blueprint(editbook) app.register_blueprint(editbook)
if oauth_available: if oauth_available:
app.register_blueprint(oauth) app.register_blueprint(oauth)
Server.startServer() success = web_server.start()
sys.exit(0 if success else 1)
if __name__ == '__main__': if __name__ == '__main__':

View File

@ -84,8 +84,8 @@ searched_ids = {}
from .worker import WorkerThread from .worker import WorkerThread
global_WorkerThread = WorkerThread() global_WorkerThread = WorkerThread()
from .server import server from .server import WebServer
Server = server() web_server = WebServer()
from .ldap import Ldap from .ldap import Ldap
ldap = Ldap() ldap = Ldap()
@ -103,7 +103,7 @@ def create_app():
Principal(app) Principal(app)
lm.init_app(app) lm.init_app(app)
app.secret_key = os.getenv('SECRET_KEY', 'A0Zr98j/3yX R~XHH!jmN]LWX/,?RT') app.secret_key = os.getenv('SECRET_KEY', 'A0Zr98j/3yX R~XHH!jmN]LWX/,?RT')
Server.init_app(app) web_server.init_app(app, config)
db.setup_db() db.setup_db()
babel.init_app(app) babel.init_app(app)
ldap.init_app(app) ldap.init_app(app)

View File

@ -41,8 +41,9 @@ from jinja2 import __version__ as jinja2Version
from pytz import __version__ as pytzVersion from pytz import __version__ as pytzVersion
from sqlalchemy import __version__ as sqlalchemyVersion from sqlalchemy import __version__ as sqlalchemyVersion
from . import db, converter, Server, uploader from . import db, converter, uploader
from .isoLanguages import __version__ as iso639Version from .isoLanguages import __version__ as iso639Version
from .server import VERSION as serverVersion
from .web import render_title_template from .web import render_title_template
@ -71,7 +72,7 @@ def stats():
versions['pySqlite'] = 'v' + db.engine.dialect.dbapi.version versions['pySqlite'] = 'v' + db.engine.dialect.dbapi.version
versions['Sqlite'] = 'v' + db.engine.dialect.dbapi.sqlite_version versions['Sqlite'] = 'v' + db.engine.dialect.dbapi.sqlite_version
versions.update(converter.versioncheck()) versions.update(converter.versioncheck())
versions.update(Server.getNameVersion()) versions.update(serverVersion)
versions['Python'] = sys.version versions['Python'] = sys.version
return render_title_template('stats.html', bookcounter=counter, authorcounter=authors, versions=versions, return render_title_template('stats.html', bookcounter=counter, authorcounter=authors, versions=versions,
categorycounter=categorys, seriecounter=series, title=_(u"Statistics"), page="stat") categorycounter=categorys, seriecounter=series, title=_(u"Statistics"), page="stat")

View File

@ -41,7 +41,7 @@ from sqlalchemy.exc import IntegrityError
from werkzeug.security import generate_password_hash from werkzeug.security import generate_password_hash
from . import constants, logger, ldap from . import constants, logger, ldap
from . import db, ub, Server, get_locale, config, updater_thread, babel, gdriveutils from . import db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils
from .helper import speaking_language, check_valid_domain, check_unrar, send_test_mail, generate_random_password, \ from .helper import speaking_language, check_valid_domain, check_unrar, send_test_mail, generate_random_password, \
send_registration_mail send_registration_mail
from .gdriveutils import is_gdrive_ready, gdrive_support, downloadFile, deleteDatabaseOnChange, listRootFolders from .gdriveutils import is_gdrive_ready, gdrive_support, downloadFile, deleteDatabaseOnChange, listRootFolders
@ -102,12 +102,10 @@ def shutdown():
showtext = {} showtext = {}
if task == 0: if task == 0:
showtext['text'] = _(u'Server restarted, please reload page') showtext['text'] = _(u'Server restarted, please reload page')
Server.setRestartTyp(True)
else: else:
showtext['text'] = _(u'Performing shutdown of server, please close window') showtext['text'] = _(u'Performing shutdown of server, please close window')
Server.setRestartTyp(False)
# stop gevent/tornado server # stop gevent/tornado server
Server.stopServer() web_server.stop(task == 0)
return json.dumps(showtext) return json.dumps(showtext)
else: else:
if task == 2: if task == 2:
@ -220,8 +218,7 @@ def view_configuration():
# ub.session.close() # ub.session.close()
# ub.engine.dispose() # ub.engine.dispose()
# stop Server # stop Server
Server.setRestartTyp(True) web_server.stop(True)
Server.stopServer()
log.info('Reboot required, restarting') log.info('Reboot required, restarting')
readColumn = db.session.query(db.Custom_Columns)\ readColumn = db.session.query(db.Custom_Columns)\
.filter(and_(db.Custom_Columns.datatype == 'bool',db.Custom_Columns.mark_for_delete == 0)).all() .filter(and_(db.Custom_Columns.datatype == 'bool',db.Custom_Columns.mark_for_delete == 0)).all()
@ -554,8 +551,7 @@ def configuration_helper(origin):
title=_(u"Basic Configuration"), page="config") title=_(u"Basic Configuration"), page="config")
if reboot_required: if reboot_required:
# stop Server # stop Server
Server.setRestartTyp(True) web_server.stop(True)
Server.stopServer()
log.info('Reboot required, restarting') log.info('Reboot required, restarting')
if origin: if origin:
success = True success = True

View File

@ -25,6 +25,7 @@ from logging.handlers import RotatingFileHandler
from .constants import BASE_DIR as _BASE_DIR from .constants import BASE_DIR as _BASE_DIR
ACCESS_FORMATTER_GEVENT = Formatter("%(message)s") ACCESS_FORMATTER_GEVENT = Formatter("%(message)s")
ACCESS_FORMATTER_TORNADO = Formatter("[%(asctime)s] %(message)s") ACCESS_FORMATTER_TORNADO = Formatter("[%(asctime)s] %(message)s")
@ -33,7 +34,6 @@ DEFAULT_LOG_LEVEL = logging.INFO
DEFAULT_LOG_FILE = os.path.join(_BASE_DIR, "calibre-web.log") DEFAULT_LOG_FILE = os.path.join(_BASE_DIR, "calibre-web.log")
DEFAULT_ACCESS_LOG = os.path.join(_BASE_DIR, "access.log") DEFAULT_ACCESS_LOG = os.path.join(_BASE_DIR, "access.log")
LOG_TO_STDERR = '/dev/stderr' LOG_TO_STDERR = '/dev/stderr'
DEFAULT_ACCESS_LEVEL= logging.INFO
logging.addLevelName(logging.WARNING, "WARN") logging.addLevelName(logging.WARNING, "WARN")
logging.addLevelName(logging.CRITICAL, "CRIT") logging.addLevelName(logging.CRITICAL, "CRIT")
@ -73,35 +73,26 @@ def is_valid_logfile(file_path):
return (not log_dir) or os.path.isdir(log_dir) return (not log_dir) or os.path.isdir(log_dir)
def setup(log_file, log_level=None, logger=None): def _absolute_log_file(log_file, default_log_file):
if logger != "access" and logger != "tornado.access":
formatter = FORMATTER
default_file = DEFAULT_LOG_FILE
else:
if logger == "tornado.access":
formatter = ACCESS_FORMATTER_TORNADO
else:
formatter = ACCESS_FORMATTER_GEVENT
default_file = DEFAULT_ACCESS_LOG
if log_file: if log_file:
if not os.path.dirname(log_file): if not os.path.dirname(log_file):
log_file = os.path.join(_BASE_DIR, log_file) log_file = os.path.join(_BASE_DIR, log_file)
log_file = os.path.abspath(log_file) return os.path.abspath(log_file)
else:
log_file = LOG_TO_STDERR return default_log_file
# log_file = default_file
def setup(log_file, log_level=None):
'''
Configure the logging output.
May be called multiple times.
'''
log_file = _absolute_log_file(log_file, DEFAULT_LOG_FILE)
# print ('%r -- %r' % (log_level, log_file))
if logger != "access" and logger != "tornado.access":
r = logging.root r = logging.root
else:
r = logging.getLogger(logger)
r.propagate = False
r.setLevel(log_level or DEFAULT_LOG_LEVEL) r.setLevel(log_level or DEFAULT_LOG_LEVEL)
previous_handler = r.handlers[0] if r.handlers else None previous_handler = r.handlers[0] if r.handlers else None
# print ('previous %r' % previous_handler)
if previous_handler: if previous_handler:
# if the log_file has not changed, don't create a new handler # if the log_file has not changed, don't create a new handler
if getattr(previous_handler, 'baseFilename', None) == log_file: if getattr(previous_handler, 'baseFilename', None) == log_file:
@ -115,16 +106,32 @@ def setup(log_file, log_level=None, logger=None):
try: try:
file_handler = RotatingFileHandler(log_file, maxBytes=50000, backupCount=2) file_handler = RotatingFileHandler(log_file, maxBytes=50000, backupCount=2)
except IOError: except IOError:
if log_file == default_file: if log_file == DEFAULT_LOG_FILE:
raise raise
file_handler = RotatingFileHandler(default_file, maxBytes=50000, backupCount=2) file_handler = RotatingFileHandler(DEFAULT_LOG_FILE, maxBytes=50000, backupCount=2)
file_handler.setFormatter(formatter) file_handler.setFormatter(FORMATTER)
for h in r.handlers: for h in r.handlers:
r.removeHandler(h) r.removeHandler(h)
h.close() h.close()
r.addHandler(file_handler) r.addHandler(file_handler)
# print ('new handler %r' % file_handler)
def create_access_log(log_file, log_name, formatter):
'''
One-time configuration for the web server's access log.
'''
log_file = _absolute_log_file(log_file, DEFAULT_ACCESS_LOG)
logging.debug("access log: %s", log_file)
access_log = logging.getLogger(log_name)
access_log.propagate = False
access_log.setLevel(logging.INFO)
file_handler = RotatingFileHandler(log_file, maxBytes=50000, backupCount=2)
file_handler.setFormatter(formatter)
access_log.addHandler(file_handler)
return access_log
# Enable logging of smtp lib debug output # Enable logging of smtp lib debug output

View File

@ -20,54 +20,55 @@
from __future__ import division, print_function, unicode_literals from __future__ import division, print_function, unicode_literals
import sys import sys
import os import os
import errno
import signal import signal
import socket import socket
import logging
try: try:
from gevent.pywsgi import WSGIServer from gevent.pywsgi import WSGIServer
from gevent.pool import Pool from gevent.pool import Pool
from gevent import __version__ as geventVersion from gevent import __version__ as _version
gevent_present = True VERSION = {'Gevent': 'v' + _version}
_GEVENT = True
except ImportError: except ImportError:
from tornado.wsgi import WSGIContainer from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop from tornado.ioloop import IOLoop
from tornado import version as tornadoVersion from tornado import version as _version
from tornado import log as tornadoLog VERSION = {'Tornado': 'v' + _version}
from tornado import options as tornadoOptions _GEVENT = False
gevent_present = False
from . import logger, config, global_WorkerThread from . import logger, global_WorkerThread
log = logger.create() log = logger.create()
class server: class WebServer:
wsgiserver = None
restart = False
app = None
access_logger = None
def __init__(self): def __init__(self):
signal.signal(signal.SIGINT, self.killServer) signal.signal(signal.SIGINT, self._killServer)
signal.signal(signal.SIGTERM, self.killServer) signal.signal(signal.SIGTERM, self._killServer)
def init_app(self, application): self.wsgiserver = None
self.app = application
self.port = config.config_port
self.listening = config.get_config_ipaddress(readable=True) + ":" + str(self.port)
self.access_logger = None self.access_logger = None
if config.config_access_log: self.restart = False
if gevent_present: self.app = None
logger.setup(config.config_access_logfile, logger.DEFAULT_ACCESS_LEVEL, "access") self.listen_address = None
self.access_logger = logging.getLogger("access") self.listen_port = None
else: self.unix_socket_file = None
logger.setup(config.config_access_logfile, logger.DEFAULT_ACCESS_LEVEL, "tornado.access")
self.ssl_args = None self.ssl_args = None
def init_app(self, application, config):
self.app = application
self.listen_address = config.get_config_ipaddress()
self.listen_port = config.config_port
if config.config_access_log:
log_name = "gevent.access" if _GEVENT else "tornado.access"
formatter = logger.ACCESS_FORMATTER_GEVENT if _GEVENT else logger.ACCESS_FORMATTER_TORNADO
self.access_logger = logger.create_access_log(config.config_access_logfile, log_name, formatter)
certfile_path = config.get_config_certfile() certfile_path = config.get_config_certfile()
keyfile_path = config.get_config_keyfile() keyfile_path = config.get_config_keyfile()
if certfile_path and keyfile_path: if certfile_path and keyfile_path:
@ -79,22 +80,46 @@ class server:
log.warning('Cert path: %s', certfile_path) log.warning('Cert path: %s', certfile_path)
log.warning('Key path: %s', keyfile_path) log.warning('Key path: %s', keyfile_path)
def _make_gevent_socket(self): def _make_gevent_unix_socket(self, socket_file):
if config.get_config_ipaddress(): # the socket file must not exist prior to bind()
return (config.get_config_ipaddress(), self.port) if os.path.exists(socket_file):
if os.name == 'nt': # avoid nuking regular files and symbolic links (could be a mistype or security issue)
return ('0.0.0.0', self.port) if os.path.isfile(socket_file) or os.path.islink(socket_file):
raise OSError(errno.EEXIST, os.strerror(errno.EEXIST), socket_file)
os.remove(socket_file)
unix_sock = WSGIServer.get_listener(socket_file, family=socket.AF_UNIX)
self.unix_socket_file = socket_file
# ensure current user and group have r/w permissions, no permissions for other users
# this way the socket can be shared in a semi-secure manner
# between the user running calibre-web and the user running the fronting webserver
os.chmod(socket_file, 0o660);
return unix_sock
def _make_gevent_socket(self):
if os.name != 'nt':
unix_socket_file = os.environ.get("CALIBRE_UNIX_SOCKET")
if unix_socket_file:
return self._make_gevent_unix_socket(unix_socket_file)
if self.listen_address:
return (self.listen_address, self.listen_port)
if os.name == 'nt':
return ('0.0.0.0', self.listen_port)
address = ('', self.listen_port)
try: try:
s = WSGIServer.get_listener(('', self.port), family=socket.AF_INET6) sock = WSGIServer.get_listener(address, family=socket.AF_INET6)
except socket.error as ex: except socket.error as ex:
log.error('%s', ex) log.error('%s', ex)
log.warning('Unable to listen on \'\', trying on IPv4 only...') log.warning('Unable to listen on \'\', trying on IPv4 only...')
s = WSGIServer.get_listener(('', self.port), family=socket.AF_INET) sock = WSGIServer.get_listener(address, family=socket.AF_INET)
log.debug("%r %r", s._sock, s._sock.getsockname()) return sock
return s
def start_gevent(self): def _start_gevent(self):
ssl_args = self.ssl_args or {} ssl_args = self.ssl_args or {}
log.info('Starting Gevent server') log.info('Starting Gevent server')
@ -102,78 +127,58 @@ class server:
sock = self._make_gevent_socket() sock = self._make_gevent_socket()
self.wsgiserver = WSGIServer(sock, self.app, log=self.access_logger, spawn=Pool(), **ssl_args) self.wsgiserver = WSGIServer(sock, self.app, log=self.access_logger, spawn=Pool(), **ssl_args)
self.wsgiserver.serve_forever() self.wsgiserver.serve_forever()
except socket.error: finally:
try: if self.unix_socket_file:
log.info('Unable to listen on "", trying on "0.0.0.0" only...') os.remove(self.unix_socket_file)
self.wsgiserver = WSGIServer(('0.0.0.0', config.config_port), self.app, spawn=Pool(), **ssl_args) self.unix_socket_file = None
self.wsgiserver.serve_forever()
except (OSError, socket.error) as e:
log.info("Error starting server: %s", e.strerror)
print("Error starting server: %s" % e.strerror)
global_WorkerThread.stop()
sys.exit(1)
except Exception:
log.exception("Unknown error while starting gevent")
sys.exit(0)
def start_tornado(self): def _start_tornado(self):
log.info('Starting Tornado server on %s', self.listening) log.info('Starting Tornado server on %s', self.listen_address)
try:
# Max Buffersize set to 200MB ) # Max Buffersize set to 200MB )
http_server = HTTPServer(WSGIContainer(self.app), http_server = HTTPServer(WSGIContainer(self.app),
max_buffer_size = 209700000, max_buffer_size = 209700000,
ssl_options=self.ssl_args) ssl_options=self.ssl_args)
address = config.get_config_ipaddress() http_server.listen(self.listen_port, self.listen_address)
http_server.listen(self.port, address)
self.wsgiserver=IOLoop.instance() self.wsgiserver=IOLoop.instance()
self.wsgiserver.start() self.wsgiserver.start()
# wait for stop signal # wait for stop signal
self.wsgiserver.close(True) self.wsgiserver.close(True)
except socket.error as err:
log.exception("Error starting tornado server")
print("Error starting server: %s" % err.strerror)
global_WorkerThread.stop()
sys.exit(1)
def startServer(self): def start(self):
if gevent_present: try:
if _GEVENT:
# leave subprocess out to allow forking for fetchers and processors # leave subprocess out to allow forking for fetchers and processors
self.start_gevent() self._start_gevent()
else: else:
self.start_tornado() self._start_tornado()
except Exception as ex:
if self.restart is True: log.error("Error starting server: %s", ex)
log.info("Performing restart of Calibre-Web") print("Error starting server: %s" % ex)
return False
finally:
self.wsgiserver = None
global_WorkerThread.stop() global_WorkerThread.stop()
if os.name == 'nt':
arguments = ["\"" + sys.executable + "\""] if not self.restart:
for e in sys.argv:
arguments.append("\"" + e + "\"")
os.execv(sys.executable, arguments)
else:
os.execl(sys.executable, sys.executable, *sys.argv)
else:
log.info("Performing shutdown of Calibre-Web") log.info("Performing shutdown of Calibre-Web")
global_WorkerThread.stop() return True
sys.exit(0)
def setRestartTyp(self,starttyp): log.info("Performing restart of Calibre-Web")
self.restart = starttyp arguments = list(sys.argv)
arguments.insert(0, sys.executable)
if os.name == 'nt':
arguments = ["\"%s\"" % a for a in arguments]
os.execv(sys.executable, arguments)
return True
def killServer(self, signum, frame): def _killServer(self, signum, frame):
self.stopServer() self.stop()
def stopServer(self): def stop(self, restart=False):
self.restart = restart
if self.wsgiserver: if self.wsgiserver:
if gevent_present: if _GEVENT:
self.wsgiserver.close() self.wsgiserver.close()
else: else:
self.wsgiserver.add_callback(self.wsgiserver.stop) self.wsgiserver.add_callback(self.wsgiserver.stop)
@staticmethod
def getNameVersion():
if gevent_present:
return {'Gevent': 'v' + geventVersion}
else:
return {'Tornado': 'v' + tornadoVersion}

View File

@ -476,28 +476,20 @@ class Config:
def get_config_certfile(self): def get_config_certfile(self):
if cli.certfilepath: if cli.certfilepath:
return cli.certfilepath return cli.certfilepath
else:
if cli.certfilepath is "": if cli.certfilepath is "":
return None return None
else:
return self.config_certfile return self.config_certfile
def get_config_keyfile(self): def get_config_keyfile(self):
if cli.keyfilepath: if cli.keyfilepath:
return cli.keyfilepath return cli.keyfilepath
else:
if cli.certfilepath is "": if cli.certfilepath is "":
return None return None
else:
return self.config_keyfile return self.config_keyfile
def get_config_ipaddress(self, readable=False): def get_config_ipaddress(self, readable=False):
if not readable: if not readable:
if cli.ipadress: return cli.ipadress or ""
return cli.ipadress
else:
return ""
else:
answer="0.0.0.0" answer="0.0.0.0"
if cli.ipadress: if cli.ipadress:
if cli.ipv6: if cli.ipv6:

View File

@ -33,7 +33,7 @@ from tempfile import gettempdir
from babel.dates import format_datetime from babel.dates import format_datetime
from flask_babel import gettext as _ from flask_babel import gettext as _
from . import constants, logger, config, get_locale, Server from . import constants, logger, config, get_locale, web_server
log = logger.create() log = logger.create()
@ -95,8 +95,7 @@ class Updater(threading.Thread):
self.status = 6 self.status = 6
log.debug(u'Preparing restart of server') log.debug(u'Preparing restart of server')
time.sleep(2) time.sleep(2)
Server.setRestartTyp(True) web_server.stop(True)
Server.stopServer()
self.status = 7 self.status = 7
time.sleep(2) time.sleep(2)
except requests.exceptions.HTTPError as ex: except requests.exceptions.HTTPError as ex: