Use google scholar to fetch metadata

This commit is contained in:
cbartondock 2021-03-31 22:16:00 -04:00
parent c0623a949c
commit 26cc64bdd6
4 changed files with 389 additions and 290 deletions

View File

@ -27,6 +27,15 @@ import json
from shutil import copyfile from shutil import copyfile
from uuid import uuid4 from uuid import uuid4
# Improve this to check if scholarly is available in a global way, like other pythonic libraries
have_scholar = True
try:
from scholarly import scholarly
except ImportError:
have_scholar = False
pass
from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, Response from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, Response
from flask_babel import gettext as _ from flask_babel import gettext as _
from flask_login import current_user, login_required from flask_login import current_user, login_required
@ -996,6 +1005,26 @@ def convert_bookformat(book_id):
flash(_(u"There was an error converting this book: %(res)s", res=rtn), category="error") flash(_(u"There was an error converting this book: %(res)s", res=rtn), category="error")
return redirect(url_for('editbook.edit_book', book_id=book_id)) return redirect(url_for('editbook.edit_book', book_id=book_id))
@editbook.route("/scholarsearch/<query>",methods=['GET'])
@login_required_if_no_ano
@edit_required
def scholar_search(query):
if have_scholar:
scholar_gen = scholarly.search_pubs(' '.join(query.split('+')))
i=0
result = []
for publication in scholar_gen:
del publication['source']
result.append(publication)
i+=1
if(i>=10):
break
return Response(json.dumps(result),mimetype='application/json')
else:
return 'Scholarly not installed'
@editbook.route("/ajax/editbooks/<param>", methods=['POST']) @editbook.route("/ajax/editbooks/<param>", methods=['POST'])
@login_required_if_no_ano @login_required_if_no_ano
@edit_required @edit_required

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 175 KiB

View File

@ -19,310 +19,375 @@
* Google Books api document: https://developers.google.com/books/docs/v1/using * Google Books api document: https://developers.google.com/books/docs/v1/using
* Douban Books api document: https://developers.douban.com/wiki/?title=book_v2 (Chinese Only) * Douban Books api document: https://developers.douban.com/wiki/?title=book_v2 (Chinese Only)
* ComicVine api document: https://comicvine.gamespot.com/api/documentation * ComicVine api document: https://comicvine.gamespot.com/api/documentation
*/ */
/* global _, i18nMsg, tinymce */ /* global _, i18nMsg, tinymce */
var dbResults = []; var dbResults = [];
var ggResults = []; var ggResults = [];
var cvResults = []; var cvResults = [];
var gsResults = [];
$(function () { $(function () {
var msg = i18nMsg; var msg = i18nMsg;
var douban = "https://api.douban.com"; var douban = "https://api.douban.com";
var dbSearch = "/v2/book/search"; var dbSearch = "/v2/book/search";
var dbDone = 0; var dbDone = 0;
var google = "https://www.googleapis.com"; var google = "https://www.googleapis.com";
var ggSearch = "/books/v1/volumes"; var ggSearch = "/books/v1/volumes";
var ggDone = 0; var ggDone = 0;
var comicvine = "https://comicvine.gamespot.com"; var comicvine = "https://comicvine.gamespot.com";
var cvSearch = "/api/search/"; var cvSearch = "/api/search/";
var cvDone = 0; var cvDone = 0;
var showFlag = 0; var googlescholar = window.location.href.split('/admin/book')[0];
var gsSearch = "/scholarsearch/"
var gsDone = 0;
var templates = { var showFlag = 0;
bookResult: _.template(
$("#template-book-result").html()
)
};
function populateForm (book) { var templates = {
tinymce.get("description").setContent(book.description); bookResult: _.template(
var uniqueTags = []; $("#template-book-result").html()
$.each(book.tags, function(i, el) { )
if ($.inArray(el, uniqueTags) === -1) uniqueTags.push(el); };
});
var ampSeparatedAuthors = (book.authors || []).join(" & "); function populateForm (book) {
$("#bookAuthor").val(ampSeparatedAuthors); tinymce.get("description").setContent(book.description);
$("#book_title").val(book.title); var uniqueTags = [];
$("#tags").val(uniqueTags.join(",")); $.each(book.tags, function(i, el) {
$("#rating").data("rating").setValue(Math.round(book.rating)); if ($.inArray(el, uniqueTags) === -1) uniqueTags.push(el);
$(".cover img").attr("src", book.cover);
$("#cover_url").val(book.cover);
$("#pubdate").val(book.publishedDate);
$("#publisher").val(book.publisher);
if (typeof book.series !== "undefined") {
$("#series").val(book.series);
}
}
function showResult () {
showFlag++;
if (showFlag === 1) {
$("#meta-info").html("<ul id=\"book-list\" class=\"media-list\"></ul>");
}
if ((ggDone === 3 || (ggDone === 1 && ggResults.length === 0)) &&
(dbDone === 3 || (dbDone === 1 && dbResults.length === 0)) &&
(cvDone === 3 || (cvDone === 1 && cvResults.length === 0))) {
$("#meta-info").html("<p class=\"text-danger\">" + msg.no_result + "</p>");
return;
}
function formatDate (date) {
var d = new Date(date),
month = "" + (d.getMonth() + 1),
day = "" + d.getDate(),
year = d.getFullYear();
if (month.length < 2) {
month = "0" + month;
}
if (day.length < 2) {
day = "0" + day;
}
return [year, month, day].join("-");
}
if (ggResults.length > 0) {
if (ggDone < 2) {
ggResults.forEach(function(result) {
var book = {
id: result.id,
title: result.volumeInfo.title,
authors: result.volumeInfo.authors || [],
description: result.volumeInfo.description || "",
publisher: result.volumeInfo.publisher || "",
publishedDate: result.volumeInfo.publishedDate || "",
tags: result.volumeInfo.categories || [],
rating: result.volumeInfo.averageRating || 0,
cover: result.volumeInfo.imageLinks ?
result.volumeInfo.imageLinks.thumbnail : location + "/../../../static/generic_cover.jpg",
url: "https://books.google.com/books?id=" + result.id,
source: {
id: "google",
description: "Google Books",
url: "https://books.google.com/"
}
};
var $book = $(templates.bookResult(book));
$book.find("img").on("click", function () {
populateForm(book);
});
$("#book-list").append($book);
});
ggDone = 2;
} else {
ggDone = 3;
}
}
if (dbResults.length > 0) {
if (dbDone < 2) {
dbResults.forEach(function(result) {
var seriesTitle = "";
if (result.series) {
seriesTitle = result.series.title;
}
var dateFomers = result.pubdate.split("-");
var publishedYear = parseInt(dateFomers[0], 10);
var publishedMonth = parseInt(dateFomers[1], 10);
var publishedDate = new Date(publishedYear, publishedMonth - 1, 1);
publishedDate = formatDate(publishedDate);
var book = {
id: result.id,
title: result.title,
authors: result.author || [],
description: result.summary,
publisher: result.publisher || "",
publishedDate: publishedDate || "",
tags: result.tags.map(function(tag) {
return tag.title.toLowerCase().replace(/,/g, "_");
}),
rating: result.rating.average || 0,
series: seriesTitle || "",
cover: result.image,
url: "https://book.douban.com/subject/" + result.id,
source: {
id: "douban",
description: "Douban Books",
url: "https://book.douban.com/"
}
};
if (book.rating > 0) {
book.rating /= 2;
}
var $book = $(templates.bookResult(book));
$book.find("img").on("click", function () {
populateForm(book);
});
$("#book-list").append($book);
});
dbDone = 2;
} else {
dbDone = 3;
}
}
if (cvResults.length > 0) {
if (cvDone < 2) {
cvResults.forEach(function(result) {
var seriesTitle = "";
if (result.volume.name) {
seriesTitle = result.volume.name;
}
var dateFomers = "";
if (result.store_date) {
dateFomers = result.store_date.split("-");
} else {
dateFomers = result.date_added.split("-");
}
var publishedYear = parseInt(dateFomers[0], 10);
var publishedMonth = parseInt(dateFomers[1], 10);
var publishedDate = new Date(publishedYear, publishedMonth - 1, 1);
publishedDate = formatDate(publishedDate);
var book = {
id: result.id,
title: seriesTitle + " #" + ("00" + result.issue_number).slice(-3) + " - " + result.name,
authors: result.author || [],
description: result.description,
publisher: "",
publishedDate: publishedDate || "",
tags: ["Comics", seriesTitle],
rating: 0,
series: seriesTitle || "",
cover: result.image.original_url,
url: result.site_detail_url,
source: {
id: "comicvine",
description: "ComicVine Books",
url: "https://comicvine.gamespot.com/"
}
};
var $book = $(templates.bookResult(book));
$book.find("img").on("click", function () {
populateForm(book);
});
$("#book-list").append($book);
});
cvDone = 2;
} else {
cvDone = 3;
}
}
}
function ggSearchBook (title) {
$.ajax({
url: google + ggSearch + "?q=" + title.replace(/\s+/gm, "+"),
type: "GET",
dataType: "jsonp",
jsonp: "callback",
success: function success(data) {
if ("items" in data) {
ggResults = data.items;
}
},
complete: function complete() {
ggDone = 1;
showResult();
$("#show-google").trigger("change");
}
});
}
function dbSearchBook (title) {
var apikey = "054022eaeae0b00e0fc068c0c0a2102a";
$.ajax({
url: douban + dbSearch + "?apikey=" + apikey + "&q=" + title + "&fields=all&count=10",
type: "GET",
dataType: "jsonp",
jsonp: "callback",
success: function success(data) {
dbResults = data.books;
},
error: function error() {
$("#meta-info").html("<p class=\"text-danger\">" + msg.search_error + "!</p>" + $("#meta-info")[0].innerHTML);
},
complete: function complete() {
dbDone = 1;
showResult();
$("#show-douban").trigger("change");
}
});
}
function cvSearchBook (title) {
var apikey = "57558043c53943d5d1e96a9ad425b0eb85532ee6";
title = encodeURIComponent(title);
$.ajax({
url: comicvine + cvSearch + "?api_key=" + apikey + "&resources=issue&query=" + title + "&sort=name:desc&format=jsonp",
type: "GET",
dataType: "jsonp",
jsonp: "json_callback",
success: function success(data) {
cvResults = data.results;
},
error: function error() {
$("#meta-info").html("<p class=\"text-danger\">" + msg.search_error + "!</p>" + $("#meta-info")[0].innerHTML);
},
complete: function complete() {
cvDone = 1;
showResult();
$("#show-comics").trigger("change");
}
});
}
function doSearch (keyword) {
showFlag = 0;
dbDone = ggDone = cvDone = 0;
dbResults = [];
ggResults = [];
cvResults = [];
$("#meta-info").text(msg.loading);
if (keyword) {
dbSearchBook(keyword);
ggSearchBook(keyword);
cvSearchBook(keyword);
}
}
$("#meta-search").on("submit", function (e) {
e.preventDefault();
var keyword = $("#keyword").val();
if (keyword) {
doSearch(keyword);
}
}); });
$("#get_meta").click(function () { var ampSeparatedAuthors = (book.authors || []).join(" & ");
var bookTitle = $("#book_title").val(); $("#bookAuthor").val(ampSeparatedAuthors);
if (bookTitle) { $("#book_title").val(book.title);
$("#keyword").val(bookTitle); $("#tags").val(uniqueTags.join(","));
doSearch(bookTitle); $("#rating").data("rating").setValue(Math.round(book.rating));
if(book.cover !== null){
$(".cover img").attr("src", book.cover);
$("#cover_url").val(book.cover);
}
$("#pubdate").val(book.publishedDate);
$("#publisher").val(book.publisher);
if (typeof book.series !== "undefined") {
$("#series").val(book.series);
}
}
function showResult () {
showFlag++;
if (showFlag === 1) {
$("#meta-info").html("<ul id=\"book-list\" class=\"media-list\"></ul>");
}
if ((ggDone === 3 || (ggDone === 1 && ggResults.length === 0)) &&
(dbDone === 3 || (dbDone === 1 && dbResults.length === 0)) &&
(cvDone === 3 || (cvDone === 1 && cvResults.length === 0)) &&
(gsDone === 3 || (gsDone === 1 && gsResults.length === 0))) {
$("#meta-info").html("<p class=\"text-danger\">" + msg.no_result + "</p>");
return;
}
function formatDate (date) {
var d = new Date(date),
month = "" + (d.getMonth() + 1),
day = "" + d.getDate(),
year = d.getFullYear();
if (month.length < 2) {
month = "0" + month;
}
if (day.length < 2) {
day = "0" + day;
}
return [year, month, day].join("-");
}
function generateID (title) {
return title.split("").reduce(function(a,b){a=((a<<5)-a)+b.charCodeAt(0);return a&a},0).toString().substr(0,12);
}
if (ggResults.length > 0) {
if (ggDone < 2) {
ggResults.forEach(function(result) {
var book = {
id: result.id,
title: result.volumeInfo.title,
authors: result.volumeInfo.authors || [],
description: result.volumeInfo.description || "",
publisher: result.volumeInfo.publisher || "",
publishedDate: result.volumeInfo.publishedDate || "",
tags: result.volumeInfo.categories || [],
rating: result.volumeInfo.averageRating || 0,
cover: result.volumeInfo.imageLinks ?
result.volumeInfo.imageLinks.thumbnail : location + "/../../../static/generic_cover.jpg",
url: "https://books.google.com/books?id=" + result.id,
source: {
id: "google",
description: "Google Books",
url: "https://books.google.com/"
}
};
var $book = $(templates.bookResult(book));
$book.find("img").on("click", function () {
populateForm(book);
});
$("#book-list").append($book);
});
ggDone = 2;
} else {
ggDone = 3;
}
}
if (gsResults.length > 0) {
if (gsDone < 2) {
gsResults.forEach(function(result) {
var book = {
id: generateID(result.bib.title),
title: result.bib.title,
authors: result.bib.author || [],
description: result.bib.abstract || "",
publisher: result.bib.venue || "",
publishedDate: result.bib.pub_year ? result.bib.pub_year+"-01-01" : "",
tags: [],
rating: 0,
series: "",
cover: null,
url: result.pub_url || result.eprint_url || "",
source: {
id: "googlescholar",
description: "Google Scholar",
link: "https://scholar.google.com/"
}
}
var $book = $(templates.bookResult(book));
$book.find("img").on("click", function () {
populateForm(book);
});
$("#book-list").append($book);
});
gsDone = 2;
}
else {
gsDone = 3;
}
}
if (dbResults.length > 0) {
if (dbDone < 2) {
dbResults.forEach(function(result) {
var seriesTitle = "";
if (result.series) {
seriesTitle = result.series.title;
}
var dateFomers = result.pubdate.split("-");
var publishedYear = parseInt(dateFomers[0], 10);
var publishedMonth = parseInt(dateFomers[1], 10);
var publishedDate = new Date(publishedYear, publishedMonth - 1, 1);
publishedDate = formatDate(publishedDate);
var book = {
id: result.id,
title: result.title,
authors: result.author || [],
description: result.summary,
publisher: result.publisher || "",
publishedDate: publishedDate || "",
tags: result.tags.map(function(tag) {
return tag.title.toLowerCase().replace(/,/g, "_");
}),
rating: result.rating.average || 0,
series: seriesTitle || "",
cover: result.image,
url: "https://book.douban.com/subject/" + result.id,
source: {
id: "douban",
description: "Douban Books",
url: "https://book.douban.com/"
}
};
if (book.rating > 0) {
book.rating /= 2;
}
var $book = $(templates.bookResult(book));
$book.find("img").on("click", function () {
populateForm(book);
});
$("#book-list").append($book);
});
dbDone = 2;
} else {
dbDone = 3;
}
}
if (cvResults.length > 0) {
if (cvDone < 2) {
cvResults.forEach(function(result) {
var seriesTitle = "";
if (result.volume.name) {
seriesTitle = result.volume.name;
}
var dateFomers = "";
if (result.store_date) {
dateFomers = result.store_date.split("-");
} else {
dateFomers = result.date_added.split("-");
}
var publishedYear = parseInt(dateFomers[0], 10);
var publishedMonth = parseInt(dateFomers[1], 10);
var publishedDate = new Date(publishedYear, publishedMonth - 1, 1);
publishedDate = formatDate(publishedDate);
var book = {
id: result.id,
title: seriesTitle + " #" + ("00" + result.issue_number).slice(-3) + " - " + result.name,
authors: result.author || [],
description: result.description,
publisher: "",
publishedDate: publishedDate || "",
tags: ["Comics", seriesTitle],
rating: 0,
series: seriesTitle || "",
cover: result.image.original_url,
url: result.site_detail_url,
source: {
id: "comicvine",
description: "ComicVine Books",
url: "https://comicvine.gamespot.com/"
}
};
var $book = $(templates.bookResult(book));
$book.find("img").on("click", function () {
populateForm(book);
});
$("#book-list").append($book);
});
cvDone = 2;
} else {
cvDone = 3;
}
}
}
function ggSearchBook (title) {
$.ajax({
url: google + ggSearch + "?q=" + title.replace(/\s+/gm, "+"),
type: "GET",
dataType: "jsonp",
jsonp: "callback",
success: function success(data) {
if ("items" in data) {
ggResults = data.items;
} }
},
complete: function complete() {
ggDone = 1;
showResult();
$("#show-google").trigger("change");
}
}); });
}
function dbSearchBook (title) {
var apikey = "054022eaeae0b00e0fc068c0c0a2102a";
$.ajax({
url: douban + dbSearch + "?apikey=" + apikey + "&q=" + title + "&fields=all&count=10",
type: "GET",
dataType: "jsonp",
jsonp: "callback",
success: function success(data) {
dbResults = data.books;
},
error: function error() {
$("#meta-info").html("<p class=\"text-danger\">" + msg.search_error + "!</p>" + $("#meta-info")[0].innerHTML);
},
complete: function complete() {
dbDone = 1;
showResult();
$("#show-douban").trigger("change");
}
});
}
function cvSearchBook (title) {
var apikey = "57558043c53943d5d1e96a9ad425b0eb85532ee6";
title = encodeURIComponent(title);
$.ajax({
url: comicvine + cvSearch + "?api_key=" + apikey + "&resources=issue&query=" + title + "&sort=name:desc&format=jsonp",
type: "GET",
dataType: "jsonp",
jsonp: "json_callback",
success: function success(data) {
cvResults = data.results;
},
error: function error() {
$("#meta-info").html("<p class=\"text-danger\">" + msg.search_error + "!</p>" + $("#meta-info")[0].innerHTML);
},
complete: function complete() {
cvDone = 1;
showResult();
$("#show-comics").trigger("change");
}
});
}
function gsSearchBook (title) {
$.ajax({
url: googlescholar + gsSearch + title.replace(/\s+/gm,'+'),
type: "GET",
dataType: "json",
success: function success(data) {
gsResults = data;
},
complete: function complete() {
gsDone = 1;
showResult();
$("#show-googlescholar").trigger("change");
}
})
}
function doSearch (keyword) {
showFlag = 0;
dbDone = ggDone = cvDone = 0;
dbResults = [];
ggResults = [];
cvResults = [];
gsResults = [];
$("#meta-info").text(msg.loading);
if (keyword) {
dbSearchBook(keyword);
ggSearchBook(keyword);
cvSearchBook(keyword);
gsSearchBook(keyword);
}
}
$("#meta-search").on("submit", function (e) {
e.preventDefault();
var keyword = $("#keyword").val();
if (keyword) {
doSearch(keyword);
}
});
$("#get_meta").click(function () {
var bookTitle = $("#book_title").val();
if (bookTitle) {
$("#keyword").val(bookTitle);
doSearch(bookTitle);
}
});
}); });

View File

@ -227,6 +227,10 @@
<input type="checkbox" id="show-comics" class="pill" data-control="comicvine" checked> <input type="checkbox" id="show-comics" class="pill" data-control="comicvine" checked>
<label for="show-comics">ComicVine <span class="glyphicon glyphicon-ok"></span></label> <label for="show-comics">ComicVine <span class="glyphicon glyphicon-ok"></span></label>
<input type="checkbox" id="show-googlescholar" class="pill" data-control="googlescholar" checked>
<label for="show-googlescholar">Google Scholar <span class="glyphicon glyphicon-ok"></span></label>
</div> </div>
<div id="meta-info"> <div id="meta-info">
@ -248,7 +252,7 @@
<img class="pull-left img-responsive" <img class="pull-left img-responsive"
data-toggle="modal" data-toggle="modal"
data-target="#metaModal" data-target="#metaModal"
src="<%= cover %>" src="<%= cover || "{{ url_for('static', filename='img/academicpaper.svg') }}" %>"
alt="Cover" alt="Cover"
> >
<div class="media-body"> <div class="media-body">