4230226716
Fixes reader button visible in detail view Fix formats to convert (added htmlz) Fix logger in updater Added request "v3" of github api on update Fix quotes parameter on external calls E-Mail logger working more stable (also on python3) Routing fixes Change import in ub
520 lines
23 KiB
Python
520 lines
23 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
|
|
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
|
|
# Copyright (C) 2018-2019 OzzieIsaacs
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
|
from cps import config, get_locale, Server, app
|
|
import threading
|
|
import zipfile
|
|
import requests
|
|
import time
|
|
from io import BytesIO
|
|
import os
|
|
import sys
|
|
import shutil
|
|
from ub import UPDATE_STABLE
|
|
from tempfile import gettempdir
|
|
import datetime
|
|
import json
|
|
from flask_babel import gettext as _
|
|
from babel.dates import format_datetime
|
|
|
|
|
|
def is_sha1(sha1):
|
|
if len(sha1) != 40:
|
|
return False
|
|
try:
|
|
int(sha1, 16)
|
|
except ValueError:
|
|
return False
|
|
return True
|
|
|
|
|
|
class Updater(threading.Thread):
|
|
|
|
def __init__(self):
|
|
threading.Thread.__init__(self)
|
|
self.status = 0
|
|
self.updateIndex = None
|
|
|
|
def get_current_version_info(self):
|
|
if config.get_update_channel == UPDATE_STABLE:
|
|
return self._stable_version_info()
|
|
else:
|
|
return self._nightly_version_info()
|
|
|
|
def get_available_updates(self, request_method):
|
|
if config.get_update_channel == UPDATE_STABLE:
|
|
return self._stable_available_updates(request_method)
|
|
else:
|
|
return self._nightly_available_updates(request_method)
|
|
|
|
def run(self):
|
|
try:
|
|
self.status = 1
|
|
app.logger.debug(u'Download update file')
|
|
headers = {'Accept': 'application/vnd.github.v3+json'}
|
|
r = requests.get(self._get_request_path(), stream=True, headers=headers)
|
|
r.raise_for_status()
|
|
|
|
self.status = 2
|
|
app.logger.debug(u'Opening zipfile')
|
|
z = zipfile.ZipFile(BytesIO(r.content))
|
|
self.status = 3
|
|
app.logger.debug(u'Extracting zipfile')
|
|
tmp_dir = gettempdir()
|
|
z.extractall(tmp_dir)
|
|
foldername = os.path.join(tmp_dir, z.namelist()[0])[:-1]
|
|
if not os.path.isdir(foldername):
|
|
self.status = 11
|
|
app.logger.info(u'Extracted contents of zipfile not found in temp folder')
|
|
return
|
|
self.status = 4
|
|
app.logger.debug(u'Replacing files')
|
|
self.update_source(foldername, config.get_main_dir)
|
|
self.status = 6
|
|
app.logger.debug(u'Preparing restart of server')
|
|
time.sleep(2)
|
|
Server.setRestartTyp(True)
|
|
Server.stopServer()
|
|
self.status = 7
|
|
time.sleep(2)
|
|
except requests.exceptions.HTTPError as ex:
|
|
app.logger.info( u'HTTP Error' + ' ' + str(ex))
|
|
self.status = 8
|
|
except requests.exceptions.ConnectionError:
|
|
app.logger.info(u'Connection error')
|
|
self.status = 9
|
|
except requests.exceptions.Timeout:
|
|
app.logger.info(u'Timeout while establishing connection')
|
|
self.status = 10
|
|
except requests.exceptions.RequestException:
|
|
self.status = 11
|
|
app.logger.info(u'General error')
|
|
|
|
def get_update_status(self):
|
|
return self.status
|
|
|
|
@classmethod
|
|
def file_to_list(self, filelist):
|
|
return [x.strip() for x in open(filelist, 'r') if not x.startswith('#EXT')]
|
|
|
|
@classmethod
|
|
def one_minus_two(self, one, two):
|
|
return [x for x in one if x not in set(two)]
|
|
|
|
@classmethod
|
|
def reduce_dirs(self, delete_files, new_list):
|
|
new_delete = []
|
|
for filename in delete_files:
|
|
parts = filename.split(os.sep)
|
|
sub = ''
|
|
for part in parts:
|
|
sub = os.path.join(sub, part)
|
|
if sub == '':
|
|
sub = os.sep
|
|
count = 0
|
|
for song in new_list:
|
|
if song.startswith(sub):
|
|
count += 1
|
|
break
|
|
if count == 0:
|
|
if sub != '\\':
|
|
new_delete.append(sub)
|
|
break
|
|
return list(set(new_delete))
|
|
|
|
@classmethod
|
|
def reduce_files(self, remove_items, exclude_items):
|
|
rf = []
|
|
for item in remove_items:
|
|
if not item.startswith(exclude_items):
|
|
rf.append(item)
|
|
return rf
|
|
|
|
@classmethod
|
|
def moveallfiles(self, root_src_dir, root_dst_dir):
|
|
change_permissions = True
|
|
if sys.platform == "win32" or sys.platform == "darwin":
|
|
change_permissions = False
|
|
else:
|
|
app.logger.debug('Update on OS-System : ' + sys.platform)
|
|
new_permissions = os.stat(root_dst_dir)
|
|
# print new_permissions
|
|
for src_dir, __, files in os.walk(root_src_dir):
|
|
dst_dir = src_dir.replace(root_src_dir, root_dst_dir, 1)
|
|
if not os.path.exists(dst_dir):
|
|
os.makedirs(dst_dir)
|
|
app.logger.debug('Create-Dir: '+dst_dir)
|
|
if change_permissions:
|
|
# print('Permissions: User '+str(new_permissions.st_uid)+' Group '+str(new_permissions.st_uid))
|
|
os.chown(dst_dir, new_permissions.st_uid, new_permissions.st_gid)
|
|
for file_ in files:
|
|
src_file = os.path.join(src_dir, file_)
|
|
dst_file = os.path.join(dst_dir, file_)
|
|
if os.path.exists(dst_file):
|
|
if change_permissions:
|
|
permission = os.stat(dst_file)
|
|
app.logger.debug('Remove file before copy: '+dst_file)
|
|
os.remove(dst_file)
|
|
else:
|
|
if change_permissions:
|
|
permission = new_permissions
|
|
shutil.move(src_file, dst_dir)
|
|
app.logger.debug('Move File '+src_file+' to '+dst_dir)
|
|
if change_permissions:
|
|
try:
|
|
os.chown(dst_file, permission.st_uid, permission.st_gid)
|
|
except (Exception) as e:
|
|
# ex = sys.exc_info()
|
|
old_permissions = os.stat(dst_file)
|
|
app.logger.debug('Fail change permissions of ' + str(dst_file) + '. Before: '
|
|
+ str(old_permissions.st_uid) + ':' + str(old_permissions.st_gid) + ' After: '
|
|
+ str(permission.st_uid) + ':' + str(permission.st_gid) + ' error: '+str(e))
|
|
return
|
|
|
|
def update_source(self, source, destination):
|
|
# destination files
|
|
old_list = list()
|
|
exclude = (
|
|
os.sep + 'app.db', os.sep + 'calibre-web.log1', os.sep + 'calibre-web.log2', os.sep + 'gdrive.db',
|
|
os.sep + 'vendor', os.sep + 'calibre-web.log', os.sep + '.git', os.sep +'client_secrets.json',
|
|
os.sep + 'gdrive_credentials', os.sep + 'settings.yaml')
|
|
for root, dirs, files in os.walk(destination, topdown=True):
|
|
for name in files:
|
|
old_list.append(os.path.join(root, name).replace(destination, ''))
|
|
for name in dirs:
|
|
old_list.append(os.path.join(root, name).replace(destination, ''))
|
|
# source files
|
|
new_list = list()
|
|
for root, dirs, files in os.walk(source, topdown=True):
|
|
for name in files:
|
|
new_list.append(os.path.join(root, name).replace(source, ''))
|
|
for name in dirs:
|
|
new_list.append(os.path.join(root, name).replace(source, ''))
|
|
|
|
delete_files = self.one_minus_two(old_list, new_list)
|
|
|
|
rf = self.reduce_files(delete_files, exclude)
|
|
|
|
remove_items = self.reduce_dirs(rf, new_list)
|
|
|
|
self.moveallfiles(source, destination)
|
|
|
|
for item in remove_items:
|
|
item_path = os.path.join(destination, item[1:])
|
|
if os.path.isdir(item_path):
|
|
app.logger.debug("Delete dir " + item_path)
|
|
shutil.rmtree(item_path, ignore_errors=True)
|
|
else:
|
|
try:
|
|
app.logger.debug("Delete file " + item_path)
|
|
# log_from_thread("Delete file " + item_path)
|
|
os.remove(item_path)
|
|
except Exception:
|
|
app.logger.debug("Could not remove:" + item_path)
|
|
shutil.rmtree(source, ignore_errors=True)
|
|
|
|
def _nightly_version_info(self):
|
|
content = {}
|
|
content[0] = '$Format:%H$'
|
|
content[1] = '$Format:%cI$'
|
|
# content[0] = 'bb7d2c6273ae4560e83950d36d64533343623a57'
|
|
# content[1] = '2018-09-09T10:13:08+02:00'
|
|
if is_sha1(content[0]) and len(content[1]) > 0:
|
|
return {'version': content[0], 'datetime': content[1]}
|
|
return False
|
|
|
|
def _stable_version_info(self):
|
|
return {'version': '0.6.1'} # Current version
|
|
|
|
def _nightly_available_updates(self, request_method):
|
|
tz = datetime.timedelta(seconds=time.timezone if (time.localtime().tm_isdst == 0) else time.altzone)
|
|
if request_method == "GET":
|
|
repository_url = 'https://api.github.com/repos/janeczku/calibre-web'
|
|
status, commit = self._load_remote_data(repository_url +'/git/refs/heads/master')
|
|
parents = []
|
|
if status['message'] != '':
|
|
return json.dumps(status)
|
|
if 'object' not in commit:
|
|
status['message'] = _(u'Unexpected data while reading update information')
|
|
return json.dumps(status)
|
|
|
|
if commit['object']['sha'] == status['current_commit_hash']:
|
|
status.update({
|
|
'update': False,
|
|
'success': True,
|
|
'message': _(u'No update available. You already have the latest version installed')
|
|
})
|
|
return json.dumps(status)
|
|
|
|
# a new update is available
|
|
status['update'] = True
|
|
|
|
try:
|
|
headers = {'Accept': 'application/vnd.github.v3+json'}
|
|
r = requests.get(repository_url + '/git/commits/' + commit['object']['sha'], headers=headers)
|
|
r.raise_for_status()
|
|
update_data = r.json()
|
|
except requests.exceptions.HTTPError as e:
|
|
status['error'] = _(u'HTTP Error') + ' ' + str(e)
|
|
except requests.exceptions.ConnectionError:
|
|
status['error'] = _(u'Connection error')
|
|
except requests.exceptions.Timeout:
|
|
status['error'] = _(u'Timeout while establishing connection')
|
|
except requests.exceptions.RequestException:
|
|
status['error'] = _(u'General error')
|
|
|
|
if status['message'] != '':
|
|
return json.dumps(status)
|
|
|
|
if 'committer' in update_data and 'message' in update_data:
|
|
status['success'] = True
|
|
status['message'] = _(
|
|
u'A new update is available. Click on the button below to update to the latest version.')
|
|
|
|
new_commit_date = datetime.datetime.strptime(
|
|
update_data['committer']['date'], '%Y-%m-%dT%H:%M:%SZ') - tz
|
|
parents.append(
|
|
[
|
|
format_datetime(new_commit_date, format='short', locale=get_locale()),
|
|
update_data['message'],
|
|
update_data['sha']
|
|
]
|
|
)
|
|
|
|
# it only makes sense to analyze the parents if we know the current commit hash
|
|
if status['current_commit_hash'] != '':
|
|
try:
|
|
parent_commit = update_data['parents'][0]
|
|
# limit the maximum search depth
|
|
remaining_parents_cnt = 10
|
|
except IndexError:
|
|
remaining_parents_cnt = None
|
|
|
|
if remaining_parents_cnt is not None:
|
|
while True:
|
|
if remaining_parents_cnt == 0:
|
|
break
|
|
|
|
# check if we are more than one update behind if so, go up the tree
|
|
if parent_commit['sha'] != status['current_commit_hash']:
|
|
try:
|
|
headers = {'Accept': 'application/vnd.github.v3+json'}
|
|
r = requests.get(parent_commit['url'], headers=headers)
|
|
r.raise_for_status()
|
|
parent_data = r.json()
|
|
|
|
parent_commit_date = datetime.datetime.strptime(
|
|
parent_data['committer']['date'], '%Y-%m-%dT%H:%M:%SZ') - tz
|
|
parent_commit_date = format_datetime(
|
|
parent_commit_date, format='short', locale=get_locale())
|
|
|
|
parents.append([parent_commit_date,
|
|
parent_data['message'].replace('\r\n','<p>').replace('\n','<p>')])
|
|
parent_commit = parent_data['parents'][0]
|
|
remaining_parents_cnt -= 1
|
|
except Exception:
|
|
# it isn't crucial if we can't get information about the parent
|
|
break
|
|
else:
|
|
# parent is our current version
|
|
break
|
|
|
|
else:
|
|
status['success'] = False
|
|
status['message'] = _(u'Could not fetch update information')
|
|
|
|
# a new update is available
|
|
status['update'] = True
|
|
if 'body' in commit:
|
|
status['success'] = True
|
|
status['message'] = _(
|
|
u'A new update is available. Click on the button below to update to the latest version.')
|
|
|
|
new_commit_date = datetime.datetime.strptime(
|
|
commit['committer']['date'], '%Y-%m-%dT%H:%M:%SZ') - tz
|
|
parents.append(
|
|
[
|
|
format_datetime(new_commit_date, format='short', locale=get_locale()),
|
|
commit['message'],
|
|
commit['sha']
|
|
]
|
|
)
|
|
|
|
# it only makes sense to analyze the parents if we know the current commit hash
|
|
if status['current_commit_hash'] != '':
|
|
try:
|
|
parent_commit = commit['parents'][0]
|
|
# limit the maximum search depth
|
|
remaining_parents_cnt = 10
|
|
except IndexError:
|
|
remaining_parents_cnt = None
|
|
|
|
if remaining_parents_cnt is not None:
|
|
while True:
|
|
if remaining_parents_cnt == 0:
|
|
break
|
|
|
|
# check if we are more than one update behind if so, go up the tree
|
|
if commit['sha'] != status['current_commit_hash']:
|
|
try:
|
|
headers = {'Accept': 'application/vnd.github.v3+json'}
|
|
r = requests.get(parent_commit['url'], headers=headers)
|
|
r.raise_for_status()
|
|
parent_data = r.json()
|
|
|
|
parent_commit_date = datetime.datetime.strptime(
|
|
parent_data['committer']['date'], '%Y-%m-%dT%H:%M:%SZ') - tz
|
|
parent_commit_date = format_datetime(
|
|
parent_commit_date, format='short', locale=get_locale())
|
|
|
|
parents.append([parent_commit_date, parent_data['message'], parent_data['sha']])
|
|
parent_commit = parent_data['parents'][0]
|
|
remaining_parents_cnt -= 1
|
|
except Exception:
|
|
# it isn't crucial if we can't get information about the parent
|
|
break
|
|
else:
|
|
# parent is our current version
|
|
break
|
|
status['history'] = parents[::-1]
|
|
return json.dumps(status)
|
|
return ''
|
|
|
|
def _stable_available_updates(self, request_method):
|
|
if request_method == "GET":
|
|
parents = []
|
|
# repository_url = 'https://api.github.com/repos/flatpak/flatpak/releases' # test URL
|
|
repository_url = 'https://api.github.com/repos/janeczku/calibre-web/releases'
|
|
status, commit = self._load_remote_data(repository_url)
|
|
if status['message'] != '':
|
|
return json.dumps(status)
|
|
if not commit:
|
|
status['success'] = True
|
|
status['message'] = _(u'No release information available')
|
|
return json.dumps(status)
|
|
version = status['current_commit_hash']
|
|
current_version = status['current_commit_hash'].split('.')
|
|
|
|
# we are already on newest version, no update available
|
|
if 'tag_name' not in commit[0]:
|
|
status['message'] = _(u'Unexpected data while reading update information')
|
|
return json.dumps(status)
|
|
if commit[0]['tag_name'] == version:
|
|
status.update({
|
|
'update': False,
|
|
'success': True,
|
|
'message': _(u'No update available. You already have the latest version installed')
|
|
})
|
|
return json.dumps(status)
|
|
|
|
i = len(commit) - 1
|
|
while i >= 0:
|
|
if 'tag_name' not in commit[i] or 'body' not in commit[i]:
|
|
status['message'] = _(u'Unexpected data while reading update information')
|
|
return json.dumps(status)
|
|
major_version_update = int(commit[i]['tag_name'].split('.')[0])
|
|
minor_version_update = int(commit[i]['tag_name'].split('.')[1])
|
|
patch_version_update = int(commit[i]['tag_name'].split('.')[2])
|
|
|
|
# Check if major versions are identical search for newest nonenqual commit and update to this one
|
|
if major_version_update == int(current_version[0]):
|
|
if (minor_version_update == int(current_version[1]) and
|
|
patch_version_update > int(current_version[2])) or \
|
|
minor_version_update > int(current_version[1]):
|
|
parents.append([commit[i]['tag_name'],commit[i]['body'].replace('\r\n', '<p>')])
|
|
i -= 1
|
|
continue
|
|
if major_version_update < int(current_version[0]):
|
|
i -= 1
|
|
continue
|
|
if major_version_update > int(current_version[0]):
|
|
# found update update to last version before major update, unless current version is on last version
|
|
# before major update
|
|
if commit[i+1]['tag_name'].split('.')[1] == current_version[1]:
|
|
parents.append([commit[i]['tag_name'],
|
|
commit[i]['body'].replace('\r\n', '<p>').replace('\n', '<p>')])
|
|
status.update({
|
|
'update': True,
|
|
'success': True,
|
|
'message': _(u'A new update is available. Click on the button below to '
|
|
u'update to version: %(version)s', version=commit[i]['tag_name']),
|
|
'history': parents
|
|
})
|
|
self.updateFile = commit[i]['zipball_url']
|
|
else:
|
|
status.update({
|
|
'update': True,
|
|
'success': True,
|
|
'message': _(u'A new update is available. Click on the button below to '
|
|
u'update to version: %(version)s', version=commit[i]['tag_name']),
|
|
'history': parents
|
|
})
|
|
self.updateFile = commit[i +1]['zipball_url']
|
|
break
|
|
if i == -1:
|
|
status.update({
|
|
'update': True,
|
|
'success': True,
|
|
'message': _(
|
|
u'A new update is available. Click on the button below to update to the latest version.'),
|
|
'history': parents
|
|
})
|
|
self.updateFile = commit[0]['zipball_url']
|
|
return json.dumps(status)
|
|
|
|
def _get_request_path(self):
|
|
if config.get_update_channel == UPDATE_STABLE:
|
|
return self.updateFile
|
|
else:
|
|
return 'https://api.github.com/repos/janeczku/calibre-web/zipball/master'
|
|
|
|
def _load_remote_data(self, repository_url):
|
|
status = {
|
|
'update': False,
|
|
'success': False,
|
|
'message': '',
|
|
'current_commit_hash': ''
|
|
}
|
|
commit = None
|
|
version = self.get_current_version_info()
|
|
if version is False:
|
|
status['current_commit_hash'] = _(u'Unknown')
|
|
else:
|
|
status['current_commit_hash'] = version['version']
|
|
try:
|
|
headers = {'Accept': 'application/vnd.github.v3+json'}
|
|
r = requests.get(repository_url, headers=headers)
|
|
commit = r.json()
|
|
r.raise_for_status()
|
|
except requests.exceptions.HTTPError as e:
|
|
if commit:
|
|
if 'message' in commit:
|
|
status['message'] = _(u'HTTP Error') + ': ' + commit['message']
|
|
else:
|
|
status['message'] = _(u'HTTP Error') + ': ' + str(e)
|
|
except requests.exceptions.ConnectionError:
|
|
status['message'] = _(u'Connection error')
|
|
except requests.exceptions.Timeout:
|
|
status['message'] = _(u'Timeout while establishing connection')
|
|
except requests.exceptions.RequestException:
|
|
status['message'] = _(u'General error')
|
|
|
|
return status, commit
|