Initial gdrive commit
Work on watching metadata More efficient storing folder keys to database Nearly completed. Need to do final touches to callback for when metadata.db updated on real server, as cannot test locally Changed callback for file changes from being hard coded to mine used url_for in template as apposed to hard coded links Fix to drive template First attempt at redownload metadata.db Fixed incorrect call to downloadFile Added logging Fixed call to copy file Added exception logging to gdriveutils + fixed string long concat Fix file download Fix backup metadata Added slashes to paths Removed threading temporarily Fix for reloading database Fix reinitialising of variables Fix check to see if custom column already setup Update to showing authenticate google drive callback + fix for reinitialising database Fixed logic for showing authenticate with google drive
This commit is contained in:
Jack Darlington
parent
f71fa5d935
commit
6d30382ae0
53
cps/db.py
53
cps/db.py
|
|
@ -12,9 +12,9 @@ import ub
|
|||
|
||||
session = None
|
||||
cc_exceptions = None
|
||||
cc_classes = None
|
||||
cc_ids = None
|
||||
books_custom_column_links = None
|
||||
cc_classes = {}
|
||||
cc_ids = []
|
||||
books_custom_column_links = {}
|
||||
engine = None
|
||||
|
||||
|
||||
|
|
@ -293,41 +293,40 @@ def setup_db():
|
|||
|
||||
cc = conn.execute("SELECT id, datatype FROM custom_columns")
|
||||
|
||||
cc_ids = []
|
||||
cc_exceptions = ['datetime', 'int', 'comments', 'float', 'composite', 'series']
|
||||
books_custom_column_links = {}
|
||||
cc_classes = {}
|
||||
for row in cc:
|
||||
if row.datatype not in cc_exceptions:
|
||||
books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link', Base.metadata,
|
||||
if row.id not in books_custom_column_links:
|
||||
books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link', Base.metadata,
|
||||
Column('book', Integer, ForeignKey('books.id'),
|
||||
primary_key=True),
|
||||
Column('value', Integer,
|
||||
ForeignKey('custom_column_' + str(row.id) + '.id'),
|
||||
primary_key=True)
|
||||
)
|
||||
cc_ids.append([row.id, row.datatype])
|
||||
if row.datatype == 'bool':
|
||||
ccdict = {'__tablename__': 'custom_column_' + str(row.id),
|
||||
'id': Column(Integer, primary_key=True),
|
||||
'book': Column(Integer, ForeignKey('books.id')),
|
||||
'value': Column(Boolean)}
|
||||
else:
|
||||
ccdict = {'__tablename__': 'custom_column_' + str(row.id),
|
||||
'id': Column(Integer, primary_key=True),
|
||||
'value': Column(String)}
|
||||
cc_classes[row.id] = type('Custom_Column_' + str(row.id), (Base,), ccdict)
|
||||
cc_ids.append([row.id, row.datatype])
|
||||
if row.datatype == 'bool':
|
||||
ccdict = {'__tablename__': 'custom_column_' + str(row.id),
|
||||
'id': Column(Integer, primary_key=True),
|
||||
'book': Column(Integer, ForeignKey('books.id')),
|
||||
'value': Column(Boolean)}
|
||||
else:
|
||||
ccdict = {'__tablename__': 'custom_column_' + str(row.id),
|
||||
'id': Column(Integer, primary_key=True),
|
||||
'value': Column(String)}
|
||||
cc_classes[row.id] = type('Custom_Column_' + str(row.id), (Base,), ccdict)
|
||||
|
||||
for id in cc_ids:
|
||||
if id[1] == 'bool':
|
||||
setattr(Books, 'custom_column_' + str(id[0]), relationship(cc_classes[id[0]],
|
||||
primaryjoin=(
|
||||
Books.id == cc_classes[id[0]].book),
|
||||
backref='books'))
|
||||
else:
|
||||
setattr(Books, 'custom_column_' + str(id[0]), relationship(cc_classes[id[0]],
|
||||
secondary=books_custom_column_links[id[0]],
|
||||
backref='books'))
|
||||
if not hasattr(Books, 'custom_column_' + str(id[0])):
|
||||
if id[1] == 'bool':
|
||||
setattr(Books, 'custom_column_' + str(id[0]), relationship(cc_classes[id[0]],
|
||||
primaryjoin=(
|
||||
Books.id == cc_classes[id[0]].book),
|
||||
backref='books'))
|
||||
else:
|
||||
setattr(Books, 'custom_column_' + str(id[0]), relationship(cc_classes[id[0]],
|
||||
secondary=books_custom_column_links[id[0]],
|
||||
backref='books'))
|
||||
|
||||
# Base.metadata.create_all(engine)
|
||||
Session = sessionmaker()
|
||||
|
|
|
|||
313
cps/gdriveutils.py
Normal file
313
cps/gdriveutils.py
Normal file
|
|
@ -0,0 +1,313 @@
|
|||
from pydrive.auth import GoogleAuth
|
||||
from pydrive.drive import GoogleDrive
|
||||
import os, time
|
||||
|
||||
from ub import config
|
||||
|
||||
from sqlalchemy import *
|
||||
from sqlalchemy import exc
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import *
|
||||
|
||||
from apiclient import errors
|
||||
|
||||
import web
|
||||
|
||||
|
||||
dbpath = os.path.join(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + os.sep + ".." + os.sep), "gdrive.db")
|
||||
engine = create_engine('sqlite:///{0}'.format(dbpath), echo=False)
|
||||
Base = declarative_base()
|
||||
|
||||
# Open session for database connection
|
||||
Session = sessionmaker()
|
||||
Session.configure(bind=engine)
|
||||
session = Session()
|
||||
|
||||
class GdriveId(Base):
|
||||
__tablename__='gdrive_ids'
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
gdrive_id = Column(Integer, unique=True)
|
||||
path = Column(String)
|
||||
|
||||
def __repr__(self):
|
||||
return str(self.path)
|
||||
|
||||
if not os.path.exists(dbpath):
|
||||
try:
|
||||
Base.metadata.create_all(engine)
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
def getDrive(gauth=None):
|
||||
if not gauth:
|
||||
gauth=GoogleAuth(settings_file='settings.yaml')
|
||||
# Try to load saved client credentials
|
||||
gauth.LoadCredentialsFile("gdrive_credentials")
|
||||
if gauth.access_token_expired:
|
||||
# Refresh them if expired
|
||||
gauth.Refresh()
|
||||
else:
|
||||
# Initialize the saved creds
|
||||
gauth.Authorize()
|
||||
# Save the current credentials to a file
|
||||
return GoogleDrive(gauth)
|
||||
|
||||
def getEbooksFolder(drive=None):
|
||||
if not drive:
|
||||
drive = getDrive()
|
||||
if drive.auth.access_token_expired:
|
||||
drive.auth.Refresh()
|
||||
ebooksFolder= "title = '%s' and 'root' in parents and mimeType = 'application/vnd.google-apps.folder' and trashed = false" % config.config_google_drive_folder
|
||||
|
||||
fileList = drive.ListFile({'q': ebooksFolder}).GetList()
|
||||
return fileList[0]
|
||||
|
||||
def getEbooksFolderId(drive=None):
|
||||
storedPathName=session.query(GdriveId).filter(GdriveId.path == '/').first()
|
||||
if storedPathName:
|
||||
return storedPathName.gdrive_id
|
||||
else:
|
||||
gDriveId=GdriveId()
|
||||
gDriveId.gdrive_id=getEbooksFolder(drive)['id']
|
||||
gDriveId.path='/'
|
||||
session.merge(gDriveId)
|
||||
session.commit()
|
||||
return
|
||||
|
||||
def getFolderInFolder(parentId, folderName, drive=None):
|
||||
if not drive:
|
||||
drive = getDrive()
|
||||
if drive.auth.access_token_expired:
|
||||
drive.auth.Refresh()
|
||||
folder= "title = '%s' and '%s' in parents and mimeType = 'application/vnd.google-apps.folder' and trashed = false" % (folderName.replace("'", "\\'"), parentId)
|
||||
fileList = drive.ListFile({'q': folder}).GetList()
|
||||
return fileList[0]
|
||||
|
||||
def getFile(pathId, fileName, drive=None):
|
||||
if not drive:
|
||||
drive = getDrive()
|
||||
if drive.auth.access_token_expired:
|
||||
drive.auth.Refresh()
|
||||
metaDataFile="'%s' in parents and trashed = false and title = '%s'" % (pathId, fileName.replace("'", "\\'"))
|
||||
|
||||
fileList = drive.ListFile({'q': metaDataFile}).GetList()
|
||||
return fileList[0]
|
||||
|
||||
def getFolderId(path, drive=None):
|
||||
if not drive:
|
||||
drive=getDrive()
|
||||
if drive.auth.access_token_expired:
|
||||
drive.auth.Refresh()
|
||||
currentFolderId=getEbooksFolderId(drive)
|
||||
sqlCheckPath=path if path[-1] =='/' else path + '/'
|
||||
storedPathName=session.query(GdriveId).filter(GdriveId.path == sqlCheckPath).first()
|
||||
|
||||
if not storedPathName:
|
||||
dbChange=False
|
||||
s=path.split('/')
|
||||
for i, x in enumerate(s):
|
||||
if len(x) > 0:
|
||||
currentPath="/".join(s[:i+1])
|
||||
if currentPath[-1] != '/':
|
||||
currentPath = currentPath + '/'
|
||||
storedPathName=session.query(GdriveId).filter(GdriveId.path == currentPath).first()
|
||||
if storedPathName:
|
||||
currentFolderId=storedPathName.gdrive_id
|
||||
else:
|
||||
currentFolderId=getFolderInFolder(currentFolderId, x, drive)['id']
|
||||
gDriveId=GdriveId()
|
||||
gDriveId.gdrive_id=currentFolderId
|
||||
gDriveId.path=currentPath
|
||||
session.merge(gDriveId)
|
||||
dbChange=True
|
||||
if dbChange:
|
||||
session.commit()
|
||||
else:
|
||||
currentFolderId=storedPathName.gdrive_id
|
||||
return currentFolderId
|
||||
|
||||
|
||||
def getFileFromEbooksFolder(drive, path, fileName):
|
||||
if not drive:
|
||||
drive=getDrive()
|
||||
if drive.auth.access_token_expired:
|
||||
drive.auth.Refresh()
|
||||
if path:
|
||||
sqlCheckPath=path if path[-1] =='/' else path + '/'
|
||||
folderId=getFolderId(path, drive)
|
||||
else:
|
||||
folderId=getEbooksFolderId(drive)
|
||||
|
||||
return getFile(folderId, fileName, drive)
|
||||
|
||||
def copyDriveFileRemote(drive, origin_file_id, copy_title):
|
||||
if not drive:
|
||||
drive=getDrive()
|
||||
if drive.auth.access_token_expired:
|
||||
drive.auth.Refresh()
|
||||
copied_file = {'title': copy_title}
|
||||
try:
|
||||
file_data = drive.auth.service.files().copy(
|
||||
fileId=origin_file_id, body=copied_file).execute()
|
||||
return drive.CreateFile({'id': file_data['id']})
|
||||
except errors.HttpError as error:
|
||||
print ('An error occurred: %s' % error)
|
||||
return None
|
||||
|
||||
def downloadFile(drive, path, filename, output):
|
||||
if not drive:
|
||||
drive=getDrive()
|
||||
if drive.auth.access_token_expired:
|
||||
drive.auth.Refresh()
|
||||
f=getFileFromEbooksFolder(drive, path, filename)
|
||||
f.GetContentFile(output)
|
||||
|
||||
def backupCalibreDbAndOptionalDownload(drive, f=None):
|
||||
pass
|
||||
if not drive:
|
||||
drive=getDrive()
|
||||
if drive.auth.access_token_expired:
|
||||
drive.auth.Refresh()
|
||||
metaDataFile="'%s' in parents and title = 'metadata.db' and trashed = false" % getEbooksFolderId()
|
||||
|
||||
fileList = drive.ListFile({'q': metaDataFile}).GetList()
|
||||
|
||||
databaseFile=fileList[0]
|
||||
|
||||
if f:
|
||||
databaseFile.GetContentFile(f)
|
||||
|
||||
def copyToDrive(drive, uploadFile, createRoot, replaceFiles,
|
||||
ignoreFiles=[],
|
||||
parent=None, prevDir=''):
|
||||
if not drive:
|
||||
drive=getDrive()
|
||||
if drive.auth.access_token_expired:
|
||||
drive.auth.Refresh()
|
||||
isInitial=not bool(parent)
|
||||
if not parent:
|
||||
parent=getEbooksFolder(drive)
|
||||
if os.path.isdir(os.path.join(prevDir,uploadFile)):
|
||||
existingFolder=drive.ListFile({'q' : "title = '%s' and '%s' in parents and trashed = false" % (os.path.basename(uploadFile), parent['id'])}).GetList()
|
||||
if len(existingFolder) == 0 and (not isInitial or createRoot):
|
||||
parent = drive.CreateFile({'title': os.path.basename(uploadFile), 'parents' : [{"kind": "drive#fileLink", 'id' : parent['id']}],
|
||||
"mimeType": "application/vnd.google-apps.folder" })
|
||||
parent.Upload()
|
||||
else:
|
||||
if (not isInitial or createRoot) and len(existingFolder) > 0:
|
||||
parent=existingFolder[0]
|
||||
for f in os.listdir(os.path.join(prevDir,uploadFile)):
|
||||
if f not in ignoreFiles:
|
||||
copyToDrive(drive, f, True, replaceFiles, ignoreFiles, parent, os.path.join(prevDir,uploadFile))
|
||||
else:
|
||||
if os.path.basename(uploadFile) not in ignoreFiles:
|
||||
existingFiles=drive.ListFile({'q' : "title = '%s' and '%s' in parents and trashed = false" % (os.path.basename(uploadFile), parent['id'])}).GetList()
|
||||
if len(existingFiles) > 0:
|
||||
driveFile = drive.CreateFile({'title': os.path.basename(uploadFile), 'parents' : [{"kind": "drive#fileLink", 'id' : parent['id']}], })
|
||||
else:
|
||||
driveFile=existingFiles[0]
|
||||
driveFile.SetContentFile(os.path.join(prevDir,uploadFile))
|
||||
driveFile.Upload()
|
||||
|
||||
def watchChange(drive, channel_id, channel_type, channel_address,
|
||||
channel_token=None, expiration=None):
|
||||
if not drive:
|
||||
drive=getDrive()
|
||||
if drive.auth.access_token_expired:
|
||||
drive.auth.Refresh()
|
||||
"""Watch for all changes to a user's Drive.
|
||||
Args:
|
||||
service: Drive API service instance.
|
||||
channel_id: Unique string that identifies this channel.
|
||||
channel_type: Type of delivery mechanism used for this channel.
|
||||
channel_address: Address where notifications are delivered.
|
||||
channel_token: An arbitrary string delivered to the target address with
|
||||
each notification delivered over this channel. Optional.
|
||||
channel_address: Address where notifications are delivered. Optional.
|
||||
Returns:
|
||||
The created channel if successful
|
||||
Raises:
|
||||
apiclient.errors.HttpError: if http request to create channel fails.
|
||||
"""
|
||||
body = {
|
||||
'id': channel_id,
|
||||
'type': channel_type,
|
||||
'address': channel_address
|
||||
}
|
||||
if channel_token:
|
||||
body['token'] = channel_token
|
||||
if expiration:
|
||||
body['expiration'] = expiration
|
||||
return drive.auth.service.changes().watch(body=body).execute()
|
||||
|
||||
def watchFile(drive, file_id, channel_id, channel_type, channel_address,
|
||||
channel_token=None, expiration=None):
|
||||
"""Watch for any changes to a specific file.
|
||||
Args:
|
||||
service: Drive API service instance.
|
||||
file_id: ID of the file to watch.
|
||||
channel_id: Unique string that identifies this channel.
|
||||
channel_type: Type of delivery mechanism used for this channel.
|
||||
channel_address: Address where notifications are delivered.
|
||||
channel_token: An arbitrary string delivered to the target address with
|
||||
each notification delivered over this channel. Optional.
|
||||
channel_address: Address where notifications are delivered. Optional.
|
||||
Returns:
|
||||
The created channel if successful
|
||||
Raises:
|
||||
apiclient.errors.HttpError: if http request to create channel fails.
|
||||
"""
|
||||
if not drive:
|
||||
drive=getDrive()
|
||||
if drive.auth.access_token_expired:
|
||||
drive.auth.Refresh()
|
||||
|
||||
body = {
|
||||
'id': channel_id,
|
||||
'type': channel_type,
|
||||
'address': channel_address
|
||||
}
|
||||
if channel_token:
|
||||
body['token'] = channel_token
|
||||
if expiration:
|
||||
body['expiration'] = expiration
|
||||
return drive.auth.service.files().watch(fileId=file_id, body=body).execute()
|
||||
|
||||
def stopChannel(drive, channel_id, resource_id):
|
||||
"""Stop watching to a specific channel.
|
||||
Args:
|
||||
service: Drive API service instance.
|
||||
channel_id: ID of the channel to stop.
|
||||
resource_id: Resource ID of the channel to stop.
|
||||
Raises:
|
||||
apiclient.errors.HttpError: if http request to create channel fails.
|
||||
"""
|
||||
if not drive:
|
||||
drive=getDrive()
|
||||
if drive.auth.access_token_expired:
|
||||
drive.auth.Refresh()
|
||||
service=drive.auth.service
|
||||
body = {
|
||||
'id': channel_id,
|
||||
'resourceId': resource_id
|
||||
}
|
||||
return drive.auth.service.channels().stop(body=body).execute()
|
||||
|
||||
def getChangeById (drive, change_id):
|
||||
if not drive:
|
||||
drive=getDrive()
|
||||
if drive.auth.access_token_expired:
|
||||
drive.auth.Refresh()
|
||||
"""Print a single Change resource information.
|
||||
|
||||
Args:
|
||||
service: Drive API service instance.
|
||||
change_id: ID of the Change resource to retrieve.
|
||||
"""
|
||||
try:
|
||||
change = drive.auth.service.changes().get(changeId=change_id).execute()
|
||||
return change
|
||||
except errors.HttpError, error:
|
||||
web.app.logger.exception(error)
|
||||
return None
|
||||
|
|
@ -7,6 +7,45 @@
|
|||
<label for="config_calibre_dir">{{_('Location of Calibre database')}}</label>
|
||||
<input type="text" class="form-control" name="config_calibre_dir" id="config_calibre_dir" value="{% if content.config_calibre_dir != None %}{{ content.config_calibre_dir }}{% endif %}" autocomplete="off">
|
||||
</div>
|
||||
<div class="form-group required">
|
||||
<label for="config_use_google_drive">{{_('Use google drive?')}}</label>
|
||||
<input type="checkbox" id="config_use_google_drive" class="form-control" name="config_use_google_drive" id="config_use_google_drive" {% if content.config_use_google_drive %}checked{% endif %} >
|
||||
</div>
|
||||
<div id="gdrive_settings">
|
||||
<div class="form-group required">
|
||||
<label for="config_google_drive_client_id">{{_('Client id')}}</label>
|
||||
<input type="text" class="form-control" name="config_google_drive_client_id" id="config_google_client_id" value="{% if content.config_google_drive_client_id %}{{content.config_google_drive_client_id}}{% endif %}" autocomplete="off">
|
||||
</div>
|
||||
<div class="form-group required">
|
||||
<label for="config_google_drive_client_secret">{{_('Client secret')}}</label>
|
||||
<input type="text" class="form-control" name="config_google_drive_client_secret" id="config_google_drive_client_secret" value="{% if content.config_google_drive_client_secret %}{{content.config_google_drive_client_secret}}{% endif %}" autocomplete="off">
|
||||
</div>
|
||||
<div class="form-group required">
|
||||
<label for="config_google_drive_calibre_url_base">{{_('Calibre Base URL')}}</label>
|
||||
<input type="text" class="form-control" name="config_google_drive_calibre_url_base" id="config_google_drive_calibre_url_base" value="{% if content.config_google_drive_calibre_url_base %}{{content.config_google_drive_calibre_url_base}}{% endif %}" autocomplete="off">
|
||||
</div>
|
||||
<div class="form-group required">
|
||||
<label for="config_google_drive_folder">{{_('Google drive Calibre folder')}}</label>
|
||||
<input type="text" class="form-control" name="config_google_drive_folder" id="config_google_drive_folder" value="{% if content.config_google_drive_folder %}{{content.config_google_drive_folder}}{% endif %}" autocomplete="off" required>
|
||||
</div>
|
||||
{% if show_authenticate_google_drive %}
|
||||
<div class="form-group required">
|
||||
<a href="{{ url_for('authenticate_google_drive') }}" class="btn btn-primary">Authenticate Google Drive</a>
|
||||
</div>
|
||||
{% else %}
|
||||
{% if content.config_google_drive_watch_changes_response %}
|
||||
<label for="config_google_drive_watch_changes_response">{{_('Metadata Watch Channel ID')}}</label>
|
||||
<div class="form-group input-group required">
|
||||
<input type="text" class="form-control" name="config_google_drive_watch_changes_response" id="config_google_drive_watch_changes_response" value="{{ content.config_google_drive_watch_changes_response['id'] }} expires on {{ content.config_google_drive_watch_changes_response['expiration'] | strftime }}" autocomplete="off" disabled="">
|
||||
<span class="input-group-btn">
|
||||
<a href="{{ url_for('revoke_watch_gdrive') }}" class="btn btn-primary">Revoke</a>
|
||||
</span>
|
||||
{% else %}
|
||||
<a href="{{ url_for('watch_gdrive') }}" class="btn btn-primary">Enable watch of metadata.db</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="config_port">{{_('Server Port')}}</label>
|
||||
<input type="number" min="1" max="65535" class="form-control" name="config_port" id="config_port" value="{% if content.config_port != None %}{{ content.config_port }}{% endif %}" autocomplete="off" required>
|
||||
|
|
@ -80,3 +119,22 @@
|
|||
</form>
|
||||
</div>
|
||||
{% endblock %}
|
||||
{% block js %}
|
||||
<script>
|
||||
$(document).ready(function() {
|
||||
$('#config_use_google_drive').trigger("change");
|
||||
});
|
||||
$('#config_use_google_drive').change(function(){
|
||||
formInputs=$("#gdrive_settings :input");
|
||||
isChecked=this.checked;
|
||||
formInputs.each(function(formInput) {
|
||||
$(this).prop('required',isChecked);
|
||||
});
|
||||
if (this.checked) {
|
||||
$('#gdrive_settings').show();
|
||||
} else {
|
||||
$('#gdrive_settings').hide();
|
||||
}
|
||||
});
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
|
|
|||
28
cps/ub.py
28
cps/ub.py
|
|
@ -11,6 +11,7 @@ import traceback
|
|||
import logging
|
||||
from werkzeug.security import generate_password_hash
|
||||
from flask_babel import gettext as _
|
||||
import json
|
||||
|
||||
dbpath = os.path.join(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + os.sep + ".." + os.sep), "app.db")
|
||||
engine = create_engine('sqlite:///{0}'.format(dbpath), echo=False)
|
||||
|
|
@ -269,6 +270,12 @@ class Settings(Base):
|
|||
config_anonbrowse = Column(SmallInteger, default=0)
|
||||
config_public_reg = Column(SmallInteger, default=0)
|
||||
config_default_role = Column(SmallInteger, default=0)
|
||||
config_use_google_drive = Column(Boolean)
|
||||
config_google_drive_client_id = Column(String)
|
||||
config_google_drive_client_secret = Column(String)
|
||||
config_google_drive_folder = Column(String)
|
||||
config_google_drive_calibre_url_base = Column(String)
|
||||
config_google_drive_watch_changes_response = Column(String)
|
||||
|
||||
def __repr__(self):
|
||||
pass
|
||||
|
|
@ -295,6 +302,16 @@ class Config:
|
|||
self.config_anonbrowse = data.config_anonbrowse
|
||||
self.config_public_reg = data.config_public_reg
|
||||
self.config_default_role = data.config_default_role
|
||||
self.config_use_google_drive = data.config_use_google_drive
|
||||
self.config_google_drive_client_id = data.config_google_drive_client_id
|
||||
self.config_google_drive_client_secret = data.config_google_drive_client_secret
|
||||
self.config_google_drive_calibre_url_base = data.config_google_drive_calibre_url_base
|
||||
self.config_google_drive_folder = data.config_google_drive_folder
|
||||
if data.config_google_drive_watch_changes_response:
|
||||
self.config_google_drive_watch_changes_response = json.loads(data.config_google_drive_watch_changes_response)
|
||||
else:
|
||||
self.config_google_drive_watch_changes_response=None
|
||||
|
||||
if self.config_calibre_dir is not None:
|
||||
self.db_configured = True
|
||||
else:
|
||||
|
|
@ -379,6 +396,17 @@ def migrate_Database():
|
|||
conn.execute("ALTER TABLE Settings ADD column `config_anonbrowse` SmallInteger DEFAULT 0")
|
||||
conn.execute("ALTER TABLE Settings ADD column `config_public_reg` SmallInteger DEFAULT 0")
|
||||
session.commit()
|
||||
|
||||
try:
|
||||
session.query(exists().where(Settings.config_use_google_drive)).scalar()
|
||||
except exc.OperationalError:
|
||||
conn = engine.connect()
|
||||
conn.execute("ALTER TABLE Settings ADD column `config_use_google_drive` INTEGER DEFAULT 0")
|
||||
conn.execute("ALTER TABLE Settings ADD column `config_google_drive_client_id` String DEFAULT ''")
|
||||
conn.execute("ALTER TABLE Settings ADD column `config_google_drive_client_secret` String DEFAULT ''")
|
||||
conn.execute("ALTER TABLE Settings ADD column `config_google_drive_calibre_url_base` INTEGER DEFAULT 0")
|
||||
conn.execute("ALTER TABLE Settings ADD column `config_google_drive_folder` String DEFAULT ''")
|
||||
conn.execute("ALTER TABLE Settings ADD column `config_google_drive_watch_changes_response` String DEFAULT ''")
|
||||
try:
|
||||
session.query(exists().where(Settings.config_default_role)).scalar()
|
||||
session.commit()
|
||||
|
|
|
|||
263
cps/web.py
263
cps/web.py
|
|
@ -1,12 +1,14 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
from pydrive.auth import GoogleAuth
|
||||
|
||||
import mimetypes
|
||||
import logging
|
||||
from logging.handlers import RotatingFileHandler
|
||||
from tempfile import gettempdir
|
||||
import textwrap
|
||||
from flask import Flask, render_template, session, request, Response, redirect, url_for, send_from_directory, \
|
||||
make_response, g, flash, abort
|
||||
make_response, g, flash, abort, send_file
|
||||
import ub
|
||||
from ub import config
|
||||
import helper
|
||||
|
|
@ -42,6 +44,15 @@ import db
|
|||
from shutil import move, copyfile
|
||||
from tornado.ioloop import IOLoop
|
||||
import StringIO
|
||||
from shutil import move
|
||||
import gdriveutils
|
||||
import io
|
||||
import hashlib
|
||||
import threading
|
||||
|
||||
import time
|
||||
|
||||
current_milli_time = lambda: int(round(time.time() * 1000))
|
||||
|
||||
try:
|
||||
from wand.image import Image
|
||||
|
|
@ -52,13 +63,67 @@ except ImportError, e:
|
|||
from cgi import escape
|
||||
|
||||
# Global variables
|
||||
gdrive_watch_callback_token='target=calibreweb-watch_files'
|
||||
global_task = None
|
||||
|
||||
def md5(fname):
|
||||
hash_md5 = hashlib.md5()
|
||||
with open(fname, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(4096), b""):
|
||||
hash_md5.update(chunk)
|
||||
return hash_md5.hexdigest()
|
||||
|
||||
class Singleton:
|
||||
"""
|
||||
A non-thread-safe helper class to ease implementing singletons.
|
||||
This should be used as a decorator -- not a metaclass -- to the
|
||||
class that should be a singleton.
|
||||
|
||||
The decorated class can define one `__init__` function that
|
||||
takes only the `self` argument. Also, the decorated class cannot be
|
||||
inherited from. Other than that, there are no restrictions that apply
|
||||
to the decorated class.
|
||||
|
||||
To get the singleton instance, use the `Instance` method. Trying
|
||||
to use `__call__` will result in a `TypeError` being raised.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, decorated):
|
||||
self._decorated = decorated
|
||||
|
||||
def Instance(self):
|
||||
"""
|
||||
Returns the singleton instance. Upon its first call, it creates a
|
||||
new instance of the decorated class and calls its `__init__` method.
|
||||
On all subsequent calls, the already created instance is returned.
|
||||
|
||||
"""
|
||||
try:
|
||||
return self._instance
|
||||
except AttributeError:
|
||||
self._instance = self._decorated()
|
||||
return self._instance
|
||||
|
||||
def __call__(self):
|
||||
raise TypeError('Singletons must be accessed through `Instance()`.')
|
||||
|
||||
def __instancecheck__(self, inst):
|
||||
return isinstance(inst, self._decorated)
|
||||
|
||||
@Singleton
|
||||
class Gauth:
|
||||
def __init__(self):
|
||||
self.auth=GoogleAuth(settings_file='settings.yaml')
|
||||
|
||||
@Singleton
|
||||
class Gdrive:
|
||||
def __init__(self):
|
||||
self.drive=gdriveutils.getDrive(Gauth.Instance().auth)
|
||||
|
||||
# Proxy Helper class
|
||||
class ReverseProxied(object):
|
||||
"""Wrap the application in this middleware and configure the
|
||||
front-end server to add these headers, to let you quietly bind
|
||||
front-end server to add these headers, to let you quietly bind
|
||||
this to a URL other than / and to an HTTP scheme that is
|
||||
different than what is used locally.
|
||||
|
||||
|
|
@ -187,6 +252,12 @@ def authenticate():
|
|||
'You have to login with proper credentials', 401,
|
||||
{'WWW-Authenticate': 'Basic realm="Login Required"'})
|
||||
|
||||
def updateGdriveCalibreFromLocal():
|
||||
gdriveutils.backupCalibreDbAndOptionalDownload(Gdrive.Instance().drive)
|
||||
gdriveutils.copyToDrive(Gdrive.Instance().drive, config.config_calibre_dir, False, True)
|
||||
for x in os.listdir(config.config_calibre_dir):
|
||||
if os.path.isdir(os.path.join(config.config_calibre_dir,x)):
|
||||
shutil.rmtree(os.path.join(config.config_calibre_dir,x))
|
||||
|
||||
def requires_basic_auth_if_no_ano(f):
|
||||
@wraps(f)
|
||||
|
|
@ -286,6 +357,17 @@ def formatdate(val):
|
|||
formatdate = datetime.datetime.strptime(conformed_timestamp[:15], "%Y%m%d %H%M%S")
|
||||
return format_date(formatdate, format='medium',locale=get_locale())
|
||||
|
||||
@app.template_filter('strftime')
|
||||
def timestamptodate(date, fmt=None):
|
||||
date=datetime.datetime.fromtimestamp(
|
||||
int(date)/1000
|
||||
)
|
||||
native = date.replace(tzinfo=None)
|
||||
if fmt:
|
||||
format=fmt
|
||||
else:
|
||||
format='%d %m %Y - %H:%S'
|
||||
return native.strftime(format)
|
||||
|
||||
def admin_required(f):
|
||||
"""
|
||||
|
|
@ -668,8 +750,15 @@ def get_opds_download_link(book_id, format):
|
|||
file_name = book.title
|
||||
if len(book.authors) > 0:
|
||||
file_name = book.authors[0].name + '-' + file_name
|
||||
file_name = helper.get_valid_filename(file_name)
|
||||
response = make_response(send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + format))
|
||||
|
||||
if config.config_use_google_drive:
|
||||
df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, book.path, '%s.%s' % (data.name, format))
|
||||
download_url = df.metadata.get('downloadUrl')
|
||||
resp, content = df.auth.Get_Http_Object().request(download_url)
|
||||
response=send_file(io.BytesIO(content))
|
||||
else:
|
||||
file_name = helper.get_valid_filename(file_name)
|
||||
response = make_response(send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + format))
|
||||
response.headers["Content-Disposition"] = "attachment; filename=\"%s.%s\"" % (data.name, format)
|
||||
return response
|
||||
|
||||
|
|
@ -802,7 +891,9 @@ def hot_books(page):
|
|||
hot_books = all_books.offset(off).limit(config.config_books_per_page)
|
||||
entries = list()
|
||||
for book in hot_books:
|
||||
entries.append(db.session.query(db.Books).filter(filter).filter(db.Books.id == book.Downloads.book_id).first())
|
||||
entry=db.session.query(db.Books).filter(filter).filter(db.Books.id == book.Downloads.book_id).first()
|
||||
if entry:
|
||||
entries.append(entry)
|
||||
numBooks = entries.__len__()
|
||||
pagination = Pagination(page, config.config_books_per_page, numBooks)
|
||||
return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
|
||||
|
|
@ -1037,6 +1128,99 @@ def stats():
|
|||
categorycounter=categorys, seriecounter=series, title=_(u"Statistics"))
|
||||
|
||||
|
||||
#@app.route("/load_gdrive")
|
||||
#@login_required
|
||||
#@admin_required
|
||||
#def load_all_gdrive_folder_ids():
|
||||
# books=db.session.query(db.Books).all()
|
||||
# for book in books:
|
||||
# gdriveutils.getFolderId(book.path, Gdrive.Instance().drive)
|
||||
# return
|
||||
|
||||
@app.route("/gdrive/authenticate")
|
||||
@login_required
|
||||
@admin_required
|
||||
def authenticate_google_drive():
|
||||
authUrl=Gauth.Instance().auth.GetAuthUrl()
|
||||
return redirect(authUrl)
|
||||
|
||||
@app.route("/gdrive/callback")
|
||||
def google_drive_callback():
|
||||
auth_code = request.args.get('code')
|
||||
credentials = Gauth.Instance().auth.flow.step2_exchange(auth_code)
|
||||
with open('gdrive_credentials' ,'w') as f:
|
||||
f.write(credentials.to_json())
|
||||
return redirect(url_for('configuration'))
|
||||
|
||||
@app.route("/gdrive/watch/subscribe")
|
||||
@login_required
|
||||
@admin_required
|
||||
def watch_gdrive():
|
||||
if not config.config_google_drive_watch_changes_response:
|
||||
address = '%scalibre-web/gdrive/watch/callback' % config.config_google_drive_calibre_url_base
|
||||
notification_id=str(uuid4())
|
||||
result = gdriveutils.watchChange(Gdrive.Instance().drive, notification_id,
|
||||
'web_hook', address, gdrive_watch_callback_token, current_milli_time() + 604800*1000)
|
||||
print (result)
|
||||
settings = ub.session.query(ub.Settings).first()
|
||||
settings.config_google_drive_watch_changes_response=json.dumps(result)
|
||||
ub.session.merge(settings)
|
||||
ub.session.commit()
|
||||
settings = ub.session.query(ub.Settings).first()
|
||||
config.loadSettings()
|
||||
|
||||
print (settings.config_google_drive_watch_changes_response)
|
||||
|
||||
return redirect(url_for('configuration'))
|
||||
|
||||
@app.route("/gdrive/watch/revoke")
|
||||
@login_required
|
||||
@admin_required
|
||||
def revoke_watch_gdrive():
|
||||
last_watch_response=config.config_google_drive_watch_changes_response
|
||||
if last_watch_response:
|
||||
response=gdriveutils.stopChannel(Gdrive.Instance().drive, last_watch_response['id'], last_watch_response['resourceId'])
|
||||
settings = ub.session.query(ub.Settings).first()
|
||||
settings.config_google_drive_watch_changes_response=None
|
||||
ub.session.merge(settings)
|
||||
ub.session.commit()
|
||||
config.loadSettings()
|
||||
return redirect(url_for('configuration'))
|
||||
|
||||
@app.route("/gdrive/watch/callback", methods=['GET', 'POST'])
|
||||
def on_received_watch_confirmation():
|
||||
app.logger.info (request.headers)
|
||||
if request.headers.get('X-Goog-Channel-Token') == gdrive_watch_callback_token \
|
||||
and request.headers.get('X-Goog-Resource-State') == 'change' \
|
||||
and request.data:
|
||||
|
||||
data=request.data
|
||||
|
||||
def updateMetaData():
|
||||
app.logger.info ('Change received from gdrive')
|
||||
app.logger.info (data)
|
||||
try:
|
||||
j=json.loads(data)
|
||||
app.logger.info ('Getting change details')
|
||||
response=gdriveutils.getChangeById(Gdrive.Instance().drive, j['id'])
|
||||
app.logger.info (response)
|
||||
if response:
|
||||
dbpath = os.path.join(config.config_calibre_dir, "metadata.db")
|
||||
if not response['deleted'] and response['file']['title'] == 'metadata.db' and response['file']['md5Checksum'] != md5(dbpath):
|
||||
app.logger.info ('Database file updated')
|
||||
copyfile (dbpath, config.config_calibre_dir + "/metadata.db_" + str(current_milli_time()))
|
||||
app.logger.info ('Backing up existing and downloading updated metadata.db')
|
||||
gdriveutils.downloadFile(Gdrive.Instance().drive, None, "metadata.db", config.config_calibre_dir + "/tmp_metadata.db")
|
||||
app.logger.info ('Setting up new DB')
|
||||
os.rename(config.config_calibre_dir + "/tmp_metadata.db", dbpath)
|
||||
db.setup_db()
|
||||
except Exception, e:
|
||||
app.logger.exception(e)
|
||||
|
||||
updateMetaData()
|
||||
return ''
|
||||
|
||||
|
||||
@app.route("/shutdown")
|
||||
@login_required
|
||||
@admin_required
|
||||
|
|
@ -1173,8 +1357,15 @@ def advanced_search():
|
|||
@app.route("/cover/<path:cover_path>")
|
||||
@login_required_if_no_ano
|
||||
def get_cover(cover_path):
|
||||
return send_from_directory(os.path.join(config.config_calibre_dir, cover_path), "cover.jpg")
|
||||
if config.config_use_google_drive:
|
||||
df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, cover_path, 'cover.jpg')
|
||||
download_url = df.metadata.get('webContentLink')
|
||||
return redirect(download_url)
|
||||
else:
|
||||
return send_from_directory(os.path.join(config.config_calibre_dir, cover_path), "cover.jpg")
|
||||
resp.headers['Content-Type']='image/jpeg'
|
||||
|
||||
return resp
|
||||
|
||||
@app.route("/opds/thumb_240_240/<path:book_id>")
|
||||
@app.route("/opds/cover_240_240/<path:book_id>")
|
||||
|
|
@ -1183,7 +1374,12 @@ def get_cover(cover_path):
|
|||
@requires_basic_auth_if_no_ano
|
||||
def feed_get_cover(book_id):
|
||||
book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
|
||||
return send_from_directory(os.path.join(config.config_calibre_dir, book.path), "cover.jpg")
|
||||
if config.config_use_google_drive:
|
||||
df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, cover_path, 'cover.jpg')
|
||||
download_url = df.metadata.get('webContentLink')
|
||||
return redirect(download_url)
|
||||
else:
|
||||
return send_from_directory(os.path.join(config.config_calibre_dir, cover_path), "cover.jpg")
|
||||
|
||||
def render_read_books(page, are_read, as_xml=False):
|
||||
readBooks=ub.session.query(ub.ReadBook).filter(ub.ReadBook.user_id == int(current_user.id)).filter(ub.ReadBook.is_read == True).all()
|
||||
|
|
@ -1308,8 +1504,13 @@ def get_download_link(book_id, format):
|
|||
if len(book.authors) > 0:
|
||||
file_name = book.authors[0].name + '-' + file_name
|
||||
file_name = helper.get_valid_filename(file_name)
|
||||
response = make_response(
|
||||
send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + format))
|
||||
if config.config_use_google_drive:
|
||||
df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, book.path, '%s.%s' % (data.name, format))
|
||||
download_url = df.metadata.get('downloadUrl')
|
||||
resp, content = df.auth.Get_Http_Object().request(download_url)
|
||||
response=send_file(io.BytesIO(content))
|
||||
else:
|
||||
response = make_response(send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + format))
|
||||
try:
|
||||
response.headers["Content-Type"] = mimetypes.types_map['.' + format]
|
||||
except:
|
||||
|
|
@ -1682,6 +1883,38 @@ def configuration_helper(origin):
|
|||
if content.config_calibre_dir != to_save["config_calibre_dir"]:
|
||||
content.config_calibre_dir = to_save["config_calibre_dir"]
|
||||
db_change = True
|
||||
##Google drive setup
|
||||
create_new_yaml=False
|
||||
if "config_google_drive_client_id" in to_save:
|
||||
if content.config_google_drive_client_id != to_save["config_google_drive_client_id"]:
|
||||
content.config_google_drive_client_id = to_save["config_google_drive_client_id"]
|
||||
create_new_yaml=True
|
||||
db_change = True
|
||||
if "config_google_drive_client_secret" in to_save:
|
||||
if content.config_google_drive_client_secret != to_save["config_google_drive_client_secret"]:
|
||||
content.config_google_drive_client_secret = to_save["config_google_drive_client_secret"]
|
||||
create_new_yaml=True
|
||||
db_change = True
|
||||
if "config_google_drive_calibre_url_base" in to_save:
|
||||
if content.config_google_drive_calibre_url_base != to_save["config_google_drive_calibre_url_base"]:
|
||||
content.config_google_drive_calibre_url_base = to_save["config_google_drive_calibre_url_base"]
|
||||
create_new_yaml=True
|
||||
db_change = True
|
||||
if ("config_use_google_drive" in to_save and not content.config_use_google_drive) or ("config_use_google_drive" not in to_save and content.config_use_google_drive):
|
||||
content.config_use_google_drive = "config_use_google_drive" in to_save
|
||||
db_change = True
|
||||
if not content.config_use_google_drive:
|
||||
create_new_yaml=False
|
||||
if create_new_yaml:
|
||||
with open('settings.yaml', 'w') as f:
|
||||
with open('gdrive_template.yaml' ,'r') as t:
|
||||
f.write(t.read() % {'client_id' : content.config_google_drive_client_id, 'client_secret' : content.config_google_drive_client_secret,
|
||||
"redirect_uri" : content.config_google_drive_calibre_url_base + 'gdrive/callback'})
|
||||
if "config_google_drive_folder" in to_save:
|
||||
if content.config_google_drive_folder != to_save["config_google_drive_folder"]:
|
||||
content.config_google_drive_folder = to_save["config_google_drive_folder"]
|
||||
db_change = True
|
||||
##
|
||||
if "config_port" in to_save:
|
||||
if content.config_port != int(to_save["config_port"]):
|
||||
content.config_port = int(to_save["config_port"])
|
||||
|
|
@ -1751,6 +1984,7 @@ def configuration_helper(origin):
|
|||
if origin:
|
||||
success = True
|
||||
return render_title_template("config_edit.html", origin=origin, success=success, content=config,
|
||||
show_authenticate_google_drive=not os.path.exists('settings.yaml') or not os.path.exists('gdrive_credentials'),
|
||||
title=_(u"Basic Configuration"))
|
||||
|
||||
|
||||
|
|
@ -1999,7 +2233,7 @@ def edit_book(book_id):
|
|||
modify_database_object(input_authors, book.authors, db.Authors, db.session, 'author')
|
||||
if author0_before_edit != book.authors[0].name:
|
||||
edited_books_id.add(book.id)
|
||||
book.author_sort=helper.get_sorted_author(input_authors[0])
|
||||
book.author_sort=helper.get_sorted_author(input_authors[0])
|
||||
|
||||
if to_save["cover_url"] and os.path.splitext(to_save["cover_url"])[1].lower() == ".jpg":
|
||||
img = requests.get(to_save["cover_url"])
|
||||
|
|
@ -2163,6 +2397,8 @@ def edit_book(book_id):
|
|||
author_names.append(author.name)
|
||||
for b in edited_books_id:
|
||||
helper.update_dir_stucture(b, config.config_calibre_dir)
|
||||
if config.config_use_google_drive:
|
||||
updateGdriveCalibreFromLocal()
|
||||
if "detail_view" in to_save:
|
||||
return redirect(url_for('show_book', id=book.id))
|
||||
else:
|
||||
|
|
@ -2227,7 +2463,7 @@ def upload():
|
|||
if is_author:
|
||||
db_author = is_author
|
||||
else:
|
||||
db_author = db.Authors(author, helper.get_sorted_author(author), "")
|
||||
db_author = db.Authors(author, helper.get_sorted_author(author), "")
|
||||
db.session.add(db_author)
|
||||
# combine path and normalize path from windows systems
|
||||
path = os.path.join(author_dir, title_dir).replace('\\','/')
|
||||
|
|
@ -2242,6 +2478,9 @@ def upload():
|
|||
author_names = []
|
||||
for author in db_book.authors:
|
||||
author_names.append(author.name)
|
||||
if config.config_use_google_drive:
|
||||
if not current_user.role_edit() and not current_user.role_admin():
|
||||
updateGdriveCalibreFromLocal()
|
||||
cc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
|
||||
if current_user.role_edit() or current_user.role_admin():
|
||||
return render_title_template('book_edit.html', book=db_book, authors=author_names, cc=cc,
|
||||
|
|
|
|||
14
gdrive_template.yaml
Normal file
14
gdrive_template.yaml
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
client_config_backend: settings
|
||||
client_config:
|
||||
client_id: %(client_id)s
|
||||
client_secret: %(client_secret)s
|
||||
redirect_uri: %(redirect_uri)s
|
||||
|
||||
save_credentials: True
|
||||
save_credentials_backend: file
|
||||
save_credentials_file: gdrive_credentials
|
||||
|
||||
get_refresh_token: True
|
||||
|
||||
oauth_scope:
|
||||
- https://www.googleapis.com/auth/drive
|
||||
Loading…
Reference in New Issue
Block a user