Merge branch 'master' into cover_thumbnail

# Conflicts:
#	test/Calibre-Web TestSummary_Linux.html
This commit is contained in:
Ozzie Isaacs 2022-04-20 07:25:37 +02:00
commit 3c98cd1b9a
63 changed files with 13803 additions and 12890 deletions

View File

@ -24,20 +24,20 @@ To receive fixes for security vulnerabilities it is required to always upgrade t
| V 0.6.13 | JavaScript could get executed in the shelf title ||
| V 0.6.13 | Login with the old session cookie after logout. Thanks to @ibarrionuevo ||
| V 0.6.14 | CSRF was possible. Thanks to @mik317 and Hagai Wechsler (WhiteSource) |CVE-2021-25965|
| V 0.6.14 | Migrated some routes to POST-requests (CSRF protection). Thanks to @scara31 ||
| V 0.6.15 | Fix for "javascript:" script links in identifier. Thanks to @scara31 ||
| V 0.6.14 | Migrated some routes to POST-requests (CSRF protection). Thanks to @scara31 |CVE-2021-4164|
| V 0.6.15 | Fix for "javascript:" script links in identifier. Thanks to @scara31 |CVE-2021-4170|
| V 0.6.15 | Cross-Site Scripting vulnerability on uploaded cover file names. Thanks to @ibarrionuevo ||
| V 0.6.15 | Creating public shelfs is now denied if user is missing the edit public shelf right. Thanks to @ibarrionuevo ||
| V 0.6.15 | Changed error message in case of trying to delete a shelf unauthorized. Thanks to @ibarrionuevo ||
| V 0.6.16 | JavaScript could get executed on authors page. Thanks to @alicaz ||
| V 0.6.16 | Localhost can no longer be used to upload covers. Thanks to @scara31 ||
| V 0.6.16 | Another case where public shelfs could be created without permission is prevented. Thanks to @nhiephon ||
| V 0.6.16 | It's prevented to get the name of a private shelfs. Thanks to @nhiephon ||
| V 0.6.17 | The SSRF Protection can no longer be bypassed via an HTTP redirect. Thanks to @416e6e61 ||
| V 0.6.17 | The SSRF Protection can no longer be bypassed via 0.0.0.0 and it's ipv6 equivalent. Thanks to @r0hanSH ||
| V 0.6.16 | JavaScript could get executed on authors page. Thanks to @alicaz |CVE-2022-0352|
| V 0.6.16 | Localhost can no longer be used to upload covers. Thanks to @scara31 |CVE-2022-0339|
| V 0.6.16 | Another case where public shelfs could be created without permission is prevented. Thanks to @nhiephon |CVE-2022-0273|
| V 0.6.16 | It's prevented to get the name of a private shelfs. Thanks to @nhiephon |CVE-2022-0405|
| V 0.6.17 | The SSRF Protection can no longer be bypassed via an HTTP redirect. Thanks to @416e6e61 |CVE-2022-0767|
| V 0.6.17 | The SSRF Protection can no longer be bypassed via 0.0.0.0 and it's ipv6 equivalent. Thanks to @r0hanSH |CVE-2022-0766|
| V 0.6.18 | Possible SQL Injection is prevented in user table Thanks to Iman Sharafaldin (Forward Security) ||
| V 0.6.18 | The SSRF protection no longer can be bypassed by IPV6/IPV4 embedding. Thanks to @416e6e61 ||
| V 0.6.18 | The SSRF protection no longer can be bypassed to connect to other servers in the local network. Thanks to @michaellrowley ||
| V 0.6.18 | The SSRF protection no longer can be bypassed by IPV6/IPV4 embedding. Thanks to @416e6e61 |CVE-2022-0939|
| V 0.6.18 | The SSRF protection no longer can be bypassed to connect to other servers in the local network. Thanks to @michaellrowley |CVE-2022-0990|
## Statement regarding Log4j (CVE-2021-44228 and related)

View File

@ -130,7 +130,9 @@ def get_comic_info(tmp_file_path, original_file_name, original_file_extension, r
series=loaded_metadata.series or "",
series_id=loaded_metadata.issue or "",
languages=loaded_metadata.language,
publisher="")
publisher="",
pubdate="",
identifiers=[])
return BookMeta(
file_path=tmp_file_path,
@ -143,4 +145,6 @@ def get_comic_info(tmp_file_path, original_file_name, original_file_extension, r
series="",
series_id="",
languages="",
publisher="")
publisher="",
pubdate="",
identifiers=[])

View File

@ -161,7 +161,7 @@ def selected_roles(dictionary):
# :rtype: BookMeta
BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, description, tags, series, '
'series_id, languages, publisher')
'series_id, languages, publisher, pubdate, identifiers')
STABLE_VERSION = {'version': '0.6.19 Beta'}

View File

@ -903,9 +903,20 @@ class CalibreDB:
.join(books_languages_link).join(Books)\
.filter(self.common_filters(return_all_languages=return_all_languages)) \
.group_by(text('books_languages_link.lang_code')).all()
tags = list()
for lang in languages:
lang[0].name = isoLanguages.get_language_name(get_locale(), lang[0].lang_code)
return sorted(languages, key=lambda x: x[0].name, reverse=reverse_order)
tag = Category(isoLanguages.get_language_name(get_locale(), lang[0].lang_code), lang[0].lang_code)
tags.append([tag, lang[1]])
# Append all books without language to list
if not return_all_languages:
no_lang_count = (self.session.query(Books)
.outerjoin(books_languages_link).outerjoin(Languages)
.filter(Languages.lang_code == None)
.filter(self.common_filters())
.count())
if no_lang_count:
tags.append([Category(_("None"), "none"), no_lang_count])
return sorted(tags, key=lambda x: x[0].name, reverse=reverse_order)
else:
if not languages:
languages = self.session.query(Languages) \
@ -977,3 +988,22 @@ def lcase(s):
_log = logger.create()
_log.error_or_exception(ex)
return s.lower()
class Category:
name = None
id = None
count = None
rating = None
def __init__(self, name, cat_id, rating=None):
self.name = name
self.id = cat_id
self.rating = rating
self.count = 1
'''class Count:
count = None
def __init__(self, count):
self.count = count'''

View File

@ -984,8 +984,13 @@ def create_book_on_upload(modify_date, meta):
# combine path and normalize path from Windows systems
path = os.path.join(author_dir, title_dir).replace('\\', '/')
try:
pubdate = datetime.strptime(meta.pubdate[:10], "%Y-%m-%d")
except:
pubdate = datetime(101, 1, 1)
# Calibre adds books with utc as timezone
db_book = db.Books(title, "", sort_authors, datetime.utcnow(), datetime(101, 1, 1),
db_book = db.Books(title, "", sort_authors, datetime.utcnow(), pubdate,
'1', datetime.utcnow(), path, meta.cover, db_author, [], "")
modify_date |= modify_database_object(input_authors, db_book.authors, db.Authors, calibre_db.session,
@ -1018,6 +1023,16 @@ def create_book_on_upload(modify_date, meta):
# flush content, get db_book.id available
calibre_db.session.flush()
# Handle identifiers now that db_book.id is available
identifier_list = []
for type_key, type_value in meta.identifiers:
identifier_list.append(db.Identifiers(type_value, type_key, db_book.id))
modification, warning = modify_identifiers(identifier_list, db_book.identifiers, calibre_db.session)
if warning:
flash(_("Identifiers are not Case Sensitive, Overwriting Old Identifier"), category="warning")
modify_date |= modification
return db_book, input_authors, title_dir, renamed_authors

View File

@ -63,13 +63,15 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
epub_metadata = {}
for s in ['title', 'description', 'creator', 'language', 'subject']:
for s in ['title', 'description', 'creator', 'language', 'subject', 'publisher', 'date']:
tmp = p.xpath('dc:%s/text()' % s, namespaces=ns)
if len(tmp) > 0:
if s == 'creator':
epub_metadata[s] = ' & '.join(split_authors(tmp))
elif s == 'subject':
epub_metadata[s] = ', '.join(tmp)
elif s == 'date':
epub_metadata[s] = tmp[0][:10]
else:
epub_metadata[s] = tmp[0]
else:
@ -78,6 +80,12 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
if epub_metadata['subject'] == 'Unknown':
epub_metadata['subject'] = ''
if epub_metadata['publisher'] == u'Unknown':
epub_metadata['publisher'] = ''
if epub_metadata['date'] == u'Unknown':
epub_metadata['date'] = ''
if epub_metadata['description'] == u'Unknown':
description = tree.xpath("//*[local-name() = 'description']/text()")
if len(description) > 0:
@ -92,6 +100,14 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
cover_file = parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path)
identifiers = []
for node in p.xpath('dc:identifier', namespaces=ns):
identifier_name=node.attrib.values()[-1];
identifier_value=node.text;
if identifier_name in ('uuid','calibre'):
continue;
identifiers.append( [identifier_name, identifier_value] )
if not epub_metadata['title']:
title = original_file_name
else:
@ -108,7 +124,9 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
series=epub_metadata['series'].encode('utf-8').decode('utf-8'),
series_id=epub_metadata['series_id'].encode('utf-8').decode('utf-8'),
languages=epub_metadata['language'],
publisher="")
publisher=epub_metadata['publisher'].encode('utf-8').decode('utf-8'),
pubdate=epub_metadata['date'],
identifiers=identifiers)
def parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path):

View File

@ -77,4 +77,6 @@ def get_fb2_info(tmp_file_path, original_file_extension):
series="",
series_id="",
languages="",
publisher="")
publisher="",
pubdate="",
identifiers=[])

View File

@ -19,11 +19,13 @@
import concurrent.futures
import requests
from bs4 import BeautifulSoup as BS # requirement
from typing import List, Optional
try:
import cchardet #optional for better speed
except ImportError:
pass
from cps import logger
from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata
import cps.logger as logger
@ -31,6 +33,9 @@ import cps.logger as logger
from operator import itemgetter
log = logger.create()
log = logger.create()
class Amazon(Metadata):
__name__ = "Amazon"
__id__ = "amazon"
@ -49,17 +54,21 @@ class Amazon(Metadata):
def search(
self, query: str, generic_cover: str = "", locale: str = "en"
):
) -> Optional[List[MetaRecord]]:
#timer=time()
def inner(link, index) -> [dict, int]:
try:
with self.session as session:
r = session.get(f"https://www.amazon.com{link}")
with self.session as session:
try:
r = session.get(f"https://www.amazon.com/{link}")
r.raise_for_status()
long_soup = BS(r.text, "lxml") #~4sec :/
soup2 = long_soup.find("div", attrs={"cel_widget_id": "dpx-books-ppd_csm_instrumentation_wrapper"})
if soup2 is None:
return
except Exception as ex:
log.warning(ex)
return
long_soup = BS(r.text, "lxml") #~4sec :/
soup2 = long_soup.find("div", attrs={"cel_widget_id": "dpx-books-ppd_csm_instrumentation_wrapper"})
if soup2 is None:
return
try:
match = MetaRecord(
title = "",
authors = "",
@ -104,27 +113,29 @@ class Amazon(Metadata):
except (AttributeError, TypeError):
match.cover = ""
return match, index
except Exception as e:
log.error_or_exception(e)
return
except Exception as e:
log.error_or_exception(e)
return
val = list()
try:
if self.active:
if self.active:
try:
results = self.session.get(
f"https://www.amazon.com/s?k={query.replace(' ', '+')}"
f"&i=digital-text&sprefix={query.replace(' ', '+')}"
f"https://www.amazon.com/s?k={query.replace(' ', '+')}&i=digital-text&sprefix={query.replace(' ', '+')}"
f"%2Cdigital-text&ref=nb_sb_noss",
headers=self.headers)
results.raise_for_status()
soup = BS(results.text, 'html.parser')
links_list = [next(filter(lambda i: "digital-text" in i["href"], x.findAll("a")))["href"] for x in
soup.findAll("div", attrs={"data-component-type": "s-search-result"})]
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
fut = {executor.submit(inner, link, index) for index, link in enumerate(links_list[:5])}
val = list(map(lambda x: x.result(), concurrent.futures.as_completed(fut)))
result = list(filter(lambda x: x, val))
return [x[0] for x in sorted(result, key=itemgetter(1))] #sort by amazons listing order for best relevance
except requests.exceptions.HTTPError as e:
log.error_or_exception(e)
return []
except requests.exceptions.HTTPError as e:
log.error_or_exception(e)
return None
except Exception as e:
log.warning(e)
return None
soup = BS(results.text, 'html.parser')
links_list = [next(filter(lambda i: "digital-text" in i["href"], x.findAll("a")))["href"] for x in
soup.findAll("div", attrs={"data-component-type": "s-search-result"})]
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
fut = {executor.submit(inner, link, index) for index, link in enumerate(links_list[:5])}
val = list(map(lambda x : x.result() ,concurrent.futures.as_completed(fut)))
result = list(filter(lambda x: x, val))
return [x[0] for x in sorted(result, key=itemgetter(1))] #sort by amazons listing order for best relevance

View File

@ -21,8 +21,11 @@ from typing import Dict, List, Optional
from urllib.parse import quote
import requests
from cps import logger
from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata
log = logger.create()
class ComicVine(Metadata):
__name__ = "ComicVine"
@ -46,10 +49,15 @@ class ComicVine(Metadata):
if title_tokens:
tokens = [quote(t.encode("utf-8")) for t in title_tokens]
query = "%20".join(tokens)
result = requests.get(
f"{ComicVine.BASE_URL}{query}{ComicVine.QUERY_PARAMS}",
headers=ComicVine.HEADERS,
)
try:
result = requests.get(
f"{ComicVine.BASE_URL}{query}{ComicVine.QUERY_PARAMS}",
headers=ComicVine.HEADERS,
)
result.raise_for_status()
except Exception as e:
log.warning(e)
return None
for result in result.json()["results"]:
match = self._parse_search_result(
result=result, generic_cover=generic_cover, locale=locale

View File

@ -0,0 +1,206 @@
# -*- coding: utf-8 -*-
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
# Copyright (C) 2022 xlivevil
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import re
from concurrent import futures
from typing import List, Optional
import requests
from html2text import HTML2Text
from lxml import etree
from cps import logger
from cps.services.Metadata import Metadata, MetaRecord, MetaSourceInfo
log = logger.create()
def html2text(html: str) -> str:
h2t = HTML2Text()
h2t.body_width = 0
h2t.single_line_break = True
h2t.emphasis_mark = "*"
return h2t.handle(html)
class Douban(Metadata):
__name__ = "豆瓣"
__id__ = "douban"
DESCRIPTION = "豆瓣"
META_URL = "https://book.douban.com/"
SEARCH_URL = "https://www.douban.com/j/search"
ID_PATTERN = re.compile(r"sid: (?P<id>\d+),")
AUTHORS_PATTERN = re.compile(r"作者|译者")
PUBLISHER_PATTERN = re.compile(r"出版社")
SUBTITLE_PATTERN = re.compile(r"副标题")
PUBLISHED_DATE_PATTERN = re.compile(r"出版年")
SERIES_PATTERN = re.compile(r"丛书")
IDENTIFIERS_PATTERN = re.compile(r"ISBN|统一书号")
TITTLE_XPATH = "//span[@property='v:itemreviewed']"
COVER_XPATH = "//a[@class='nbg']"
INFO_XPATH = "//*[@id='info']//span[@class='pl']"
TAGS_XPATH = "//a[contains(@class, 'tag')]"
DESCRIPTION_XPATH = "//div[@id='link-report']//div[@class='intro']"
RATING_XPATH = "//div[@class='rating_self clearfix']/strong"
session = requests.Session()
session.headers = {
'user-agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36 Edg/98.0.1108.56',
}
def search(
self, query: str, generic_cover: str = "", locale: str = "en"
) -> Optional[List[MetaRecord]]:
if self.active:
log.debug(f"starting search {query} on douban")
if title_tokens := list(
self.get_title_tokens(query, strip_joiners=False)
):
query = "+".join(title_tokens)
try:
r = self.session.get(
self.SEARCH_URL, params={"cat": 1001, "q": query}
)
r.raise_for_status()
except Exception as e:
log.warning(e)
return None
results = r.json()
if results["total"] == 0:
return []
book_id_list = [
self.ID_PATTERN.search(item).group("id")
for item in results["items"][:10] if self.ID_PATTERN.search(item)
]
with futures.ThreadPoolExecutor(max_workers=5) as executor:
fut = [
executor.submit(self._parse_single_book, book_id, generic_cover)
for book_id in book_id_list
]
val = [
future.result()
for future in futures.as_completed(fut) if future.result()
]
return val
def _parse_single_book(
self, id: str, generic_cover: str = ""
) -> Optional[MetaRecord]:
url = f"https://book.douban.com/subject/{id}/"
try:
r = self.session.get(url)
r.raise_for_status()
except Exception as e:
log.warning(e)
return None
match = MetaRecord(
id=id,
title="",
authors=[],
url=url,
source=MetaSourceInfo(
id=self.__id__,
description=self.DESCRIPTION,
link=self.META_URL,
),
)
html = etree.HTML(r.content.decode("utf8"))
match.title = html.xpath(self.TITTLE_XPATH)[0].text
match.cover = html.xpath(self.COVER_XPATH)[0].attrib["href"] or generic_cover
try:
rating_num = float(html.xpath(self.RATING_XPATH)[0].text.strip())
except Exception:
rating_num = 0
match.rating = int(-1 * rating_num // 2 * -1) if rating_num else 0
tag_elements = html.xpath(self.TAGS_XPATH)
if len(tag_elements):
match.tags = [tag_element.text for tag_element in tag_elements]
description_element = html.xpath(self.DESCRIPTION_XPATH)
if len(description_element):
match.description = html2text(etree.tostring(
description_element[-1], encoding="utf8").decode("utf8"))
info = html.xpath(self.INFO_XPATH)
for element in info:
text = element.text
if self.AUTHORS_PATTERN.search(text):
next = element.getnext()
while next is not None and next.tag != "br":
match.authors.append(next.text)
next = next.getnext()
elif self.PUBLISHER_PATTERN.search(text):
match.publisher = element.tail.strip()
elif self.SUBTITLE_PATTERN.search(text):
match.title = f'{match.title}:' + element.tail.strip()
elif self.PUBLISHED_DATE_PATTERN.search(text):
match.publishedDate = self._clean_date(element.tail.strip())
elif self.SUBTITLE_PATTERN.search(text):
match.series = element.getnext().text
elif i_type := self.IDENTIFIERS_PATTERN.search(text):
match.identifiers[i_type.group()] = element.tail.strip()
return match
def _clean_date(self, date: str) -> str:
"""
Clean up the date string to be in the format YYYY-MM-DD
Examples of possible patterns:
'2014-7-16', '1988年4月', '1995-04', '2021-8', '2020-12-1', '1996年',
'1972', '2004/11/01', '1959年3月北京第1版第1印'
"""
year = date[:4]
moon = "01"
day = "01"
if len(date) > 5:
digit = []
ls = []
for i in range(5, len(date)):
if date[i].isdigit():
digit.append(date[i])
elif digit:
ls.append("".join(digit) if len(digit)==2 else f"0{digit[0]}")
digit = []
if digit:
ls.append("".join(digit) if len(digit)==2 else f"0{digit[0]}")
moon = ls[0]
if len(ls)>1:
day = ls[1]
return f"{year}-{moon}-{day}"

View File

@ -22,9 +22,12 @@ from urllib.parse import quote
import requests
from cps import logger
from cps.isoLanguages import get_lang3, get_language_name
from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata
log = logger.create()
class Google(Metadata):
__name__ = "Google"
@ -45,7 +48,12 @@ class Google(Metadata):
if title_tokens:
tokens = [quote(t.encode("utf-8")) for t in title_tokens]
query = "+".join(tokens)
results = requests.get(Google.SEARCH_URL + query)
try:
results = requests.get(Google.SEARCH_URL + query)
results.raise_for_status()
except Exception as e:
log.warning(e)
return None
for result in results.json().get("items", []):
val.append(
self._parse_search_result(

View File

@ -27,9 +27,12 @@ from html2text import HTML2Text
from lxml.html import HtmlElement, fromstring, tostring
from markdown2 import Markdown
from cps import logger
from cps.isoLanguages import get_language_name
from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata
log = logger.create()
SYMBOLS_TO_TRANSLATE = (
"öÖüÜóÓőŐúÚéÉáÁűŰíÍąĄćĆęĘłŁńŃóÓśŚźŹżŻ",
"oOuUoOoOuUeEaAuUiIaAcCeElLnNoOsSzZzZ",
@ -112,20 +115,23 @@ class LubimyCzytac(Metadata):
self, query: str, generic_cover: str = "", locale: str = "en"
) -> Optional[List[MetaRecord]]:
if self.active:
result = requests.get(self._prepare_query(title=query))
if result.text:
root = fromstring(result.text)
lc_parser = LubimyCzytacParser(root=root, metadata=self)
matches = lc_parser.parse_search_results()
if matches:
with ThreadPool(processes=10) as pool:
final_matches = pool.starmap(
lc_parser.parse_single_book,
[(match, generic_cover, locale) for match in matches],
)
return final_matches
return matches
return []
try:
result = requests.get(self._prepare_query(title=query))
result.raise_for_status()
except Exception as e:
log.warning(e)
return None
root = fromstring(result.text)
lc_parser = LubimyCzytacParser(root=root, metadata=self)
matches = lc_parser.parse_search_results()
if matches:
with ThreadPool(processes=10) as pool:
final_matches = pool.starmap(
lc_parser.parse_single_book,
[(match, generic_cover, locale) for match in matches],
)
return final_matches
return matches
def _prepare_query(self, title: str) -> str:
query = ""
@ -202,7 +208,12 @@ class LubimyCzytacParser:
def parse_single_book(
self, match: MetaRecord, generic_cover: str, locale: str
) -> MetaRecord:
response = requests.get(match.url)
try:
response = requests.get(match.url)
response.raise_for_status()
except Exception as e:
log.warning(e)
return None
self.root = fromstring(response.text)
match.cover = self._parse_cover(generic_cover=generic_cover)
match.description = self._parse_description()

View File

@ -28,8 +28,12 @@ try:
except FakeUserAgentError:
raise ImportError("No module named 'scholarly'")
from cps import logger
from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata
log = logger.create()
class scholar(Metadata):
__name__ = "Google Scholar"
__id__ = "googlescholar"
@ -44,7 +48,11 @@ class scholar(Metadata):
if title_tokens:
tokens = [quote(t.encode("utf-8")) for t in title_tokens]
query = " ".join(tokens)
scholar_gen = itertools.islice(scholarly.search_pubs(query), 10)
try:
scholar_gen = itertools.islice(scholarly.search_pubs(query), 10)
except Exception as e:
log.warning(e)
return None
for result in scholar_gen:
match = self._parse_search_result(
result=result, generic_cover="", locale=locale

View File

@ -57,7 +57,7 @@ for f in modules:
try:
importlib.import_module("cps.metadata_provider." + a)
new_list.append(a)
except ImportError as e:
except (ImportError, IndentationError, SyntaxError) as e:
log.error("Import error for metadata source: {} - {}".format(a, e))
pass
@ -138,6 +138,6 @@ def metadata_search():
if active.get(c.__id__, True)
}
for future in concurrent.futures.as_completed(meta):
data.extend([asdict(x) for x in future.result()])
data.extend([asdict(x) for x in future.result() if x])
# log.info({'Time elapsed {}'.format(current_milli_time()-start)})
return Response(json.dumps(data), mimetype="application/json")

View File

@ -33,7 +33,7 @@ $(".datepicker").datepicker({
if (results) {
pubDate = new Date(results[1], parseInt(results[2], 10) - 1, results[3]) || new Date(this.value);
$(this).next('input')
.val(pubDate.toLocaleDateString(language))
.val(pubDate.toLocaleDateString(language.replaceAll("_","-")))
.removeClass("hidden");
}
}).trigger("change");

View File

@ -92,14 +92,19 @@ $(function () {
data: {"query": keyword},
dataType: "json",
success: function success(data) {
$("#meta-info").html("<ul id=\"book-list\" class=\"media-list\"></ul>");
data.forEach(function(book) {
var $book = $(templates.bookResult(book));
$book.find("img").on("click", function () {
populateForm(book);
if (data.length) {
$("#meta-info").html("<ul id=\"book-list\" class=\"media-list\"></ul>");
data.forEach(function(book) {
var $book = $(templates.bookResult(book));
$book.find("img").on("click", function () {
populateForm(book);
});
$("#book-list").append($book);
});
$("#book-list").append($book);
});
}
else {
$("#meta-info").html("<p class=\"text-danger\">" + msg.no_result + "!</p>" + $("#meta-info")[0].innerHTML)
}
},
error: function error() {
$("#meta-info").html("<p class=\"text-danger\">" + msg.search_error + "!</p>" + $("#meta-info")[0].innerHTML);

View File

@ -1,35 +0,0 @@
{% extends "layout.html" %}
{% block body %}
<h1>{{title}}</h1>
<div class="filterheader hidden-xs">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
<div id="asc" data-order="{{ order }}" data-id="{{ data }}" class="btn btn-primary {% if order == 1 %} active{% endif%}"><span class="glyphicon glyphicon-sort-by-alphabet"></span></div>
<div id="desc" data-id="{{ data }}" class="btn btn-primary{% if order == 0 %} active{% endif%}"><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></div>
{% if charlist|length %}
<div id="all" class="active btn btn-primary {% if charlist|length > 9 %}hidden-sm{% endif %}">{{_('All')}}</div>
{% endif %}
<div class="btn-group character {% if charlist|length > 9 %}hidden-sm{% endif %}" role="group">
{% for char in charlist%}
<div class="btn btn-primary char">{{char}}</div>
{% endfor %}
</div>
</div>
<div class="container">
<div div id="list" class="col-xs-12 col-sm-6">
{% for lang in languages %}
{% if loop.index0 == (loop.length/2)|int and loop.length > 20 %}
</div>
<div id="second" class="col-xs-12 col-sm-6">
{% endif %}
<div class="row" data-id="{{lang[0].name}}">
<div class="col-xs-2 col-sm-2 col-md-1" align="left"><span class="badge">{{lang[1]}}</span></div>
<div class="col-xs-10 col-sm-10 col-md-11"><a id="list_{{loop.index0}}" href="{{url_for('web.books_list', book_id=lang[0].lang_code, data=data, sort_param='stored')}}">{{lang[0].name}}</a></div>
</div>
{% endfor %}
</div>
</div>
{% endblock %}
{% block js %}
<script src="{{ url_for('static', filename='js/filter_list.js') }}"></script>
{% endblock %}

View File

@ -14,7 +14,7 @@
{% endif %}
<div class="btn-group character {% if charlist|length > 9 %}hidden-sm{% endif %}" role="group">
{% for char in charlist%}
<div class="btn btn-primary char">{{char.char}}</div>
<div class="btn btn-primary char">{{char[0]}}</div>
{% endfor %}
</div>
@ -29,8 +29,8 @@
</div>
<div id="second" class="col-xs-12 col-sm-6">
{% endif %}
<div class="row" {% if entry[0].sort %}data-name="{{entry[0].name}}"{% endif %} data-id="{% if entry[0].sort %}{{entry[0].sort}}{% else %}{% if entry.name %}{{entry.name}}{% else %}{{entry[0].name}}{% endif %}{% endif %}">
<div class="col-xs-2 col-sm-2 col-md-1" align="left"><span class="badge">{{entry.count}}</span></div>
<div class="row" {% if entry[0].sort %}data-name="{{entry[0].name}}"{% endif %} data-id="{% if entry[0].sort %}{{entry[0].sort}}{% else %}{% if entry[0].format %}{{entry[0].format}}{% else %}{% if entry[0].rating %}{{entry[0].rating}}{% else %}{{entry[0].name}}{% endif %}{% endif %}{% endif %}">
<div class="col-xs-2 col-sm-2 col-md-1" align="left"><span class="badge">{{entry[1]}}</span></div>
<div class="col-xs-10 col-sm-10 col-md-11"><a id="list_{{loop.index0}}" href="{% if entry.format %}{{url_for('web.books_list', data=data, sort_param='stored', book_id=entry.format )}}{% else %}{{url_for('web.books_list', data=data, sort_param='stored', book_id=entry[0].id )}}{% endif %}">
{% if entry.name %}
<div class="rating">

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -107,52 +107,10 @@ def default_meta(tmp_file_path, original_file_name, original_file_extension):
series="",
series_id="",
languages="",
publisher="")
def parse_xmp(pdf_file):
"""
Parse XMP Metadata and prepare for BookMeta object
"""
try:
xmp_info = pdf_file.getXmpMetadata()
except Exception as ex:
log.debug('Can not read XMP metadata {}'.format(ex))
return None
if xmp_info:
try:
xmp_author = xmp_info.dc_creator # list
except AttributeError:
xmp_author = ['']
if xmp_info.dc_title:
xmp_title = xmp_info.dc_title['x-default']
else:
xmp_title = ''
if xmp_info.dc_description:
xmp_description = xmp_info.dc_description['x-default']
else:
xmp_description = ''
languages = []
try:
for i in xmp_info.dc_language:
#calibre-web currently only takes one language.
languages.append(isoLanguages.get_lang3(i))
except AttributeError:
languages.append('')
xmp_tags = ', '.join(xmp_info.dc_subject)
xmp_publisher = ', '.join(xmp_info.dc_publisher)
return {'author': xmp_author,
'title': xmp_title,
'subject': xmp_description,
'tags': xmp_tags, 'languages': languages,
'publisher': xmp_publisher
}
publisher="",
pubdate="",
identifiers=[]
)
def parse_xmp(pdf_file):
@ -251,7 +209,9 @@ def pdf_meta(tmp_file_path, original_file_name, original_file_extension):
series="",
series_id="",
languages=','.join(languages),
publisher=publisher)
publisher=publisher,
pubdate="",
identifiers=[])
def pdf_preview(tmp_file_path, tmp_dir):

View File

@ -307,10 +307,20 @@ def get_matching_tags():
return json_dumps
def generate_char_list(data_colum, db_link):
return (calibre_db.session.query(func.upper(func.substr(data_colum, 1, 1)).label('char'))
def generate_char_list(entries): # data_colum, db_link):
char_list = list()
for entry in entries:
upper_char = entry[0].name[0].upper()
if upper_char not in char_list:
char_list.append(upper_char)
return char_list
def query_char_list(data_colum, db_link):
results = (calibre_db.session.query(func.upper(func.substr(data_colum, 1, 1)).label('char'))
.join(db_link).join(db.Books).filter(calibre_db.common_filters())
.group_by(func.upper(func.substr(data_colum, 1, 1))).all())
return results
def get_sort_function(sort_param, data):
@ -526,50 +536,92 @@ def render_author_books(page, author_id, order):
def render_publisher_books(page, book_id, order):
publisher = calibre_db.session.query(db.Publishers).filter(db.Publishers.id == book_id).first()
if publisher:
if book_id == '-1':
entries, random, pagination = calibre_db.fill_indexpage(page, 0,
db.Books,
db.Books.publishers.any(db.Publishers.id == book_id),
db.Publishers.name == None,
[db.Series.name, order[0][0], db.Books.series_index],
True, config.config_read_column,
db.books_publishers_link,
db.Books.id == db.books_publishers_link.c.book,
db.Publishers,
db.books_series_link,
db.Books.id == db.books_series_link.c.book,
db.Series)
publisher = _("None")
else:
publisher = calibre_db.session.query(db.Publishers).filter(db.Publishers.id == book_id).first()
if publisher:
entries, random, pagination = calibre_db.fill_indexpage(page, 0,
db.Books,
db.Books.publishers.any(
db.Publishers.id == book_id),
[db.Series.name, order[0][0],
db.Books.series_index],
True, config.config_read_column,
db.books_series_link,
db.Books.id == db.books_series_link.c.book,
db.Series)
publisher = publisher.name
else:
abort(404)
return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id,
title=_(u"Publisher: %(name)s", name=publisher),
page="publisher",
order=order[1])
def render_series_books(page, book_id, order):
if book_id == '-1':
entries, random, pagination = calibre_db.fill_indexpage(page, 0,
db.Books,
db.Series.name == None,
[order[0][0]],
True, config.config_read_column,
db.books_series_link,
db.Books.id == db.books_series_link.c.book,
db.Series)
return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id,
title=_(u"Publisher: %(name)s", name=publisher.name),
page="publisher",
order=order[1])
series_name = _("None")
else:
abort(404)
def render_series_books(page, book_id, order):
name = calibre_db.session.query(db.Series).filter(db.Series.id == book_id).first()
if name:
entries, random, pagination = calibre_db.fill_indexpage(page, 0,
db.Books,
db.Books.series.any(db.Series.id == book_id),
[order[0][0]],
True, config.config_read_column)
return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
title=_(u"Series: %(serie)s", serie=name.name), page="series", order=order[1])
else:
abort(404)
series_name = calibre_db.session.query(db.Series).filter(db.Series.id == book_id).first()
if series_name:
entries, random, pagination = calibre_db.fill_indexpage(page, 0,
db.Books,
db.Books.series.any(db.Series.id == book_id),
[order[0][0]],
True, config.config_read_column)
series_name = series_name.name
else:
abort(404)
return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
title=_(u"Series: %(serie)s", serie=series_name), page="series", order=order[1])
def render_ratings_books(page, book_id, order):
name = calibre_db.session.query(db.Ratings).filter(db.Ratings.id == book_id).first()
entries, random, pagination = calibre_db.fill_indexpage(page, 0,
db.Books,
db.Books.ratings.any(db.Ratings.id == book_id),
[order[0][0]],
True, config.config_read_column)
if name and name.rating <= 10:
if book_id == '-1':
entries, random, pagination = calibre_db.fill_indexpage(page, 0,
db.Books,
db.Books.ratings == None,
[order[0][0]],
True, config.config_read_column,
db.books_series_link,
db.Books.id == db.books_series_link.c.book,
db.Series)
title = _(u"Rating: None")
rating = -1
else:
name = calibre_db.session.query(db.Ratings).filter(db.Ratings.id == book_id).first()
entries, random, pagination = calibre_db.fill_indexpage(page, 0,
db.Books,
db.Books.ratings.any(db.Ratings.id == book_id),
[order[0][0]],
True, config.config_read_column)
title = _(u"Rating: %(rating)s stars", rating=int(name.rating / 2))
rating = name.rating
if title and rating <= 10:
return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
title=_(u"Rating: %(rating)s stars", rating=int(name.rating / 2)),
page="ratings",
order=order[1])
title=title, page="ratings", order=order[1])
else:
abort(404)
@ -591,33 +643,61 @@ def render_formats_books(page, book_id, order):
def render_category_books(page, book_id, order):
name = calibre_db.session.query(db.Tags).filter(db.Tags.id == book_id).first()
if name:
if book_id == '-1':
entries, random, pagination = calibre_db.fill_indexpage(page, 0,
db.Books,
db.Books.tags.any(db.Tags.id == book_id),
db.Tags.name == None,
[order[0][0], db.Series.name, db.Books.series_index],
True, config.config_read_column,
db.books_tags_link,
db.Books.id == db.books_tags_link.c.book,
db.Tags,
db.books_series_link,
db.Books.id == db.books_series_link.c.book,
db.Series)
return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id,
title=_(u"Category: %(name)s", name=name.name), page="category", order=order[1])
tagsname = _("None")
else:
abort(404)
tagsname = calibre_db.session.query(db.Tags).filter(db.Tags.id == book_id).first()
if tagsname:
entries, random, pagination = calibre_db.fill_indexpage(page, 0,
db.Books,
db.Books.tags.any(db.Tags.id == book_id),
[order[0][0], db.Series.name,
db.Books.series_index],
True, config.config_read_column,
db.books_series_link,
db.Books.id == db.books_series_link.c.book,
db.Series)
tagsname = tagsname.name
else:
abort(404)
return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id,
title=_(u"Category: %(name)s", name=tagsname), page="category", order=order[1])
def render_language_books(page, name, order):
try:
lang_name = isoLanguages.get_language_name(get_locale(), name)
if name.lower() != "none":
lang_name = isoLanguages.get_language_name(get_locale(), name)
else:
lang_name = _("None")
except KeyError:
abort(404)
entries, random, pagination = calibre_db.fill_indexpage(page, 0,
db.Books,
db.Books.languages.any(db.Languages.lang_code == name),
[order[0][0]],
True, config.config_read_column)
if name == "none":
entries, random, pagination = calibre_db.fill_indexpage(page, 0,
db.Books,
db.Languages.lang_code == None,
[order[0][0]],
True, config.config_read_column,
db.books_languages_link,
db.Books.id == db.books_languages_link.c.book,
db.Languages)
else:
entries, random, pagination = calibre_db.fill_indexpage(page, 0,
db.Books,
db.Books.languages.any(db.Languages.lang_code == name),
[order[0][0]],
True, config.config_read_column)
return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name,
title=_(u"Language: %(name)s", name=lang_name), page="language", order=order[1])
@ -880,7 +960,7 @@ def author_list():
entries = calibre_db.session.query(db.Authors, func.count('books_authors_link.book').label('count')) \
.join(db.books_authors_link).join(db.Books).filter(calibre_db.common_filters()) \
.group_by(text('books_authors_link.author')).order_by(order).all()
char_list = generate_char_list(db.Authors.sort, db.books_authors_link)
char_list = query_char_list(db.Authors.sort, db.books_authors_link)
# If not creating a copy, readonly databases can not display authornames with "|" in it as changing the name
# starts a change session
author_copy = copy.deepcopy(entries)
@ -926,7 +1006,15 @@ def publisher_list():
entries = calibre_db.session.query(db.Publishers, func.count('books_publishers_link.book').label('count')) \
.join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \
.group_by(text('books_publishers_link.publisher')).order_by(order).all()
char_list = generate_char_list(db.Publishers.name, db.books_publishers_link)
no_publisher_count = (calibre_db.session.query(db.Books)
.outerjoin(db.books_publishers_link).outerjoin(db.Publishers)
.filter(db.Publishers.name == None)
.filter(calibre_db.common_filters())
.count())
if no_publisher_count:
entries.append([db.Category(_("None"), "-1"), no_publisher_count])
entries = sorted(entries, key=lambda x: x[0].name, reverse=not order_no)
char_list = generate_char_list(entries)
return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=char_list,
title=_(u"Publishers"), page="publisherlist", data="publisher", order=order_no)
else:
@ -943,11 +1031,19 @@ def series_list():
else:
order = db.Series.sort.asc()
order_no = 1
char_list = generate_char_list(db.Series.sort, db.books_series_link)
char_list = query_char_list(db.Series.sort, db.books_series_link)
if current_user.get_view_property('series', 'series_view') == 'list':
entries = calibre_db.session.query(db.Series, func.count('books_series_link.book').label('count')) \
.join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \
.group_by(text('books_series_link.series')).order_by(order).all()
no_series_count = (calibre_db.session.query(db.Books)
.outerjoin(db.books_series_link).outerjoin(db.Series)
.filter(db.Series.name == None)
.filter(calibre_db.common_filters())
.count())
if no_series_count:
entries.append([db.Category(_("None"), "-1"), no_series_count])
entries = sorted(entries, key=lambda x: x[0].name, reverse=not order_no)
return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=char_list,
title=_(u"Series"), page="serieslist", data="series", order=order_no)
else:
@ -976,6 +1072,13 @@ def ratings_list():
(db.Ratings.rating / 2).label('name')) \
.join(db.books_ratings_link).join(db.Books).filter(calibre_db.common_filters()) \
.group_by(text('books_ratings_link.rating')).order_by(order).all()
no_rating_count = (calibre_db.session.query(db.Books)
.outerjoin(db.books_ratings_link).outerjoin(db.Ratings)
.filter(db.Ratings.rating == None)
.filter(calibre_db.common_filters())
.count())
entries.append([db.Category(_("None"), "-1", -1), no_rating_count])
entries = sorted(entries, key=lambda x: x[0].rating, reverse=not order_no)
return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=list(),
title=_(u"Ratings list"), page="ratingslist", data="ratings", order=order_no)
else:
@ -997,6 +1100,12 @@ def formats_list():
db.Data.format.label('format')) \
.join(db.Books).filter(calibre_db.common_filters()) \
.group_by(db.Data.format).order_by(order).all()
no_format_count = (calibre_db.session.query(db.Books).outerjoin(db.Data)
.filter(db.Data.format == None)
.filter(calibre_db.common_filters())
.count())
if no_format_count:
entries.append([db.Category(_("None"), "-1"), no_format_count])
return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=list(),
title=_(u"File formats list"), page="formatslist", data="formats", order=order_no)
else:
@ -1008,15 +1117,10 @@ def formats_list():
def language_overview():
if current_user.check_visibility(constants.SIDEBAR_LANGUAGE) and current_user.filter_language() == u"all":
order_no = 0 if current_user.get_view_property('language', 'dir') == 'desc' else 1
char_list = list()
languages = calibre_db.speaking_language(reverse_order=not order_no, with_count=True)
for lang in languages:
upper_lang = lang[0].name[0].upper()
if upper_lang not in char_list:
char_list.append(upper_lang)
return render_title_template('languages.html', languages=languages,
charlist=char_list, title=_(u"Languages"), page="langlist",
data="language", order=order_no)
char_list = generate_char_list(languages)
return render_title_template('list.html', entries=languages, folder='web.books_list', charlist=char_list,
title=_(u"Languages"), page="langlist", data="language", order=order_no)
else:
abort(404)
@ -1034,7 +1138,15 @@ def category_list():
entries = calibre_db.session.query(db.Tags, func.count('books_tags_link.book').label('count')) \
.join(db.books_tags_link).join(db.Books).order_by(order).filter(calibre_db.common_filters()) \
.group_by(text('books_tags_link.tag')).all()
char_list = generate_char_list(db.Tags.name, db.books_tags_link)
no_tag_count = (calibre_db.session.query(db.Books)
.outerjoin(db.books_tags_link).outerjoin(db.Tags)
.filter(db.Tags.name == None)
.filter(calibre_db.common_filters())
.count())
if no_tag_count:
entries.append([db.Category(_("None"), "-1"), no_tag_count])
entries = sorted(entries, key=lambda x: x[0].name, reverse=not order_no)
char_list = generate_char_list(entries)
return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=char_list,
title=_(u"Categories"), page="catlist", data="category", order=order_no)
else:

File diff suppressed because it is too large Load Diff

View File

@ -2,7 +2,7 @@ APScheduler>=3.6.3,<3.10.0
werkzeug<2.1.0
Babel>=1.3,<3.0
Flask-Babel>=0.11.1,<2.1.0
Flask-Login>=0.3.2,<0.5.1
Flask-Login>=0.3.2,<0.6.1
Flask-Principal>=0.3.2,<0.5.1
backports_abc>=0.4
Flask>=1.0.2,<2.1.0

View File

@ -42,7 +42,7 @@ install_requires =
werkzeug<2.1.0
Babel>=1.3,<3.0
Flask-Babel>=0.11.1,<2.1.0
Flask-Login>=0.3.2,<0.5.1
Flask-Login>=0.3.2,<0.6.1
Flask-Principal>=0.3.2,<0.5.1
backports_abc>=0.4
Flask>=1.0.2,<2.1.0