6d30382ae0
Work on watching metadata More efficient storing folder keys to database Nearly completed. Need to do final touches to callback for when metadata.db updated on real server, as cannot test locally Changed callback for file changes from being hard coded to mine used url_for in template as apposed to hard coded links Fix to drive template First attempt at redownload metadata.db Fixed incorrect call to downloadFile Added logging Fixed call to copy file Added exception logging to gdriveutils + fixed string long concat Fix file download Fix backup metadata Added slashes to paths Removed threading temporarily Fix for reloading database Fix reinitialising of variables Fix check to see if custom column already setup Update to showing authenticate google drive callback + fix for reinitialising database Fixed logic for showing authenticate with google drive
314 lines
11 KiB
Python
314 lines
11 KiB
Python
from pydrive.auth import GoogleAuth
|
|
from pydrive.drive import GoogleDrive
|
|
import os, time
|
|
|
|
from ub import config
|
|
|
|
from sqlalchemy import *
|
|
from sqlalchemy import exc
|
|
from sqlalchemy.ext.declarative import declarative_base
|
|
from sqlalchemy.orm import *
|
|
|
|
from apiclient import errors
|
|
|
|
import web
|
|
|
|
|
|
dbpath = os.path.join(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + os.sep + ".." + os.sep), "gdrive.db")
|
|
engine = create_engine('sqlite:///{0}'.format(dbpath), echo=False)
|
|
Base = declarative_base()
|
|
|
|
# Open session for database connection
|
|
Session = sessionmaker()
|
|
Session.configure(bind=engine)
|
|
session = Session()
|
|
|
|
class GdriveId(Base):
|
|
__tablename__='gdrive_ids'
|
|
|
|
id = Column(Integer, primary_key=True)
|
|
gdrive_id = Column(Integer, unique=True)
|
|
path = Column(String)
|
|
|
|
def __repr__(self):
|
|
return str(self.path)
|
|
|
|
if not os.path.exists(dbpath):
|
|
try:
|
|
Base.metadata.create_all(engine)
|
|
except Exception:
|
|
raise
|
|
|
|
def getDrive(gauth=None):
|
|
if not gauth:
|
|
gauth=GoogleAuth(settings_file='settings.yaml')
|
|
# Try to load saved client credentials
|
|
gauth.LoadCredentialsFile("gdrive_credentials")
|
|
if gauth.access_token_expired:
|
|
# Refresh them if expired
|
|
gauth.Refresh()
|
|
else:
|
|
# Initialize the saved creds
|
|
gauth.Authorize()
|
|
# Save the current credentials to a file
|
|
return GoogleDrive(gauth)
|
|
|
|
def getEbooksFolder(drive=None):
|
|
if not drive:
|
|
drive = getDrive()
|
|
if drive.auth.access_token_expired:
|
|
drive.auth.Refresh()
|
|
ebooksFolder= "title = '%s' and 'root' in parents and mimeType = 'application/vnd.google-apps.folder' and trashed = false" % config.config_google_drive_folder
|
|
|
|
fileList = drive.ListFile({'q': ebooksFolder}).GetList()
|
|
return fileList[0]
|
|
|
|
def getEbooksFolderId(drive=None):
|
|
storedPathName=session.query(GdriveId).filter(GdriveId.path == '/').first()
|
|
if storedPathName:
|
|
return storedPathName.gdrive_id
|
|
else:
|
|
gDriveId=GdriveId()
|
|
gDriveId.gdrive_id=getEbooksFolder(drive)['id']
|
|
gDriveId.path='/'
|
|
session.merge(gDriveId)
|
|
session.commit()
|
|
return
|
|
|
|
def getFolderInFolder(parentId, folderName, drive=None):
|
|
if not drive:
|
|
drive = getDrive()
|
|
if drive.auth.access_token_expired:
|
|
drive.auth.Refresh()
|
|
folder= "title = '%s' and '%s' in parents and mimeType = 'application/vnd.google-apps.folder' and trashed = false" % (folderName.replace("'", "\\'"), parentId)
|
|
fileList = drive.ListFile({'q': folder}).GetList()
|
|
return fileList[0]
|
|
|
|
def getFile(pathId, fileName, drive=None):
|
|
if not drive:
|
|
drive = getDrive()
|
|
if drive.auth.access_token_expired:
|
|
drive.auth.Refresh()
|
|
metaDataFile="'%s' in parents and trashed = false and title = '%s'" % (pathId, fileName.replace("'", "\\'"))
|
|
|
|
fileList = drive.ListFile({'q': metaDataFile}).GetList()
|
|
return fileList[0]
|
|
|
|
def getFolderId(path, drive=None):
|
|
if not drive:
|
|
drive=getDrive()
|
|
if drive.auth.access_token_expired:
|
|
drive.auth.Refresh()
|
|
currentFolderId=getEbooksFolderId(drive)
|
|
sqlCheckPath=path if path[-1] =='/' else path + '/'
|
|
storedPathName=session.query(GdriveId).filter(GdriveId.path == sqlCheckPath).first()
|
|
|
|
if not storedPathName:
|
|
dbChange=False
|
|
s=path.split('/')
|
|
for i, x in enumerate(s):
|
|
if len(x) > 0:
|
|
currentPath="/".join(s[:i+1])
|
|
if currentPath[-1] != '/':
|
|
currentPath = currentPath + '/'
|
|
storedPathName=session.query(GdriveId).filter(GdriveId.path == currentPath).first()
|
|
if storedPathName:
|
|
currentFolderId=storedPathName.gdrive_id
|
|
else:
|
|
currentFolderId=getFolderInFolder(currentFolderId, x, drive)['id']
|
|
gDriveId=GdriveId()
|
|
gDriveId.gdrive_id=currentFolderId
|
|
gDriveId.path=currentPath
|
|
session.merge(gDriveId)
|
|
dbChange=True
|
|
if dbChange:
|
|
session.commit()
|
|
else:
|
|
currentFolderId=storedPathName.gdrive_id
|
|
return currentFolderId
|
|
|
|
|
|
def getFileFromEbooksFolder(drive, path, fileName):
|
|
if not drive:
|
|
drive=getDrive()
|
|
if drive.auth.access_token_expired:
|
|
drive.auth.Refresh()
|
|
if path:
|
|
sqlCheckPath=path if path[-1] =='/' else path + '/'
|
|
folderId=getFolderId(path, drive)
|
|
else:
|
|
folderId=getEbooksFolderId(drive)
|
|
|
|
return getFile(folderId, fileName, drive)
|
|
|
|
def copyDriveFileRemote(drive, origin_file_id, copy_title):
|
|
if not drive:
|
|
drive=getDrive()
|
|
if drive.auth.access_token_expired:
|
|
drive.auth.Refresh()
|
|
copied_file = {'title': copy_title}
|
|
try:
|
|
file_data = drive.auth.service.files().copy(
|
|
fileId=origin_file_id, body=copied_file).execute()
|
|
return drive.CreateFile({'id': file_data['id']})
|
|
except errors.HttpError as error:
|
|
print ('An error occurred: %s' % error)
|
|
return None
|
|
|
|
def downloadFile(drive, path, filename, output):
|
|
if not drive:
|
|
drive=getDrive()
|
|
if drive.auth.access_token_expired:
|
|
drive.auth.Refresh()
|
|
f=getFileFromEbooksFolder(drive, path, filename)
|
|
f.GetContentFile(output)
|
|
|
|
def backupCalibreDbAndOptionalDownload(drive, f=None):
|
|
pass
|
|
if not drive:
|
|
drive=getDrive()
|
|
if drive.auth.access_token_expired:
|
|
drive.auth.Refresh()
|
|
metaDataFile="'%s' in parents and title = 'metadata.db' and trashed = false" % getEbooksFolderId()
|
|
|
|
fileList = drive.ListFile({'q': metaDataFile}).GetList()
|
|
|
|
databaseFile=fileList[0]
|
|
|
|
if f:
|
|
databaseFile.GetContentFile(f)
|
|
|
|
def copyToDrive(drive, uploadFile, createRoot, replaceFiles,
|
|
ignoreFiles=[],
|
|
parent=None, prevDir=''):
|
|
if not drive:
|
|
drive=getDrive()
|
|
if drive.auth.access_token_expired:
|
|
drive.auth.Refresh()
|
|
isInitial=not bool(parent)
|
|
if not parent:
|
|
parent=getEbooksFolder(drive)
|
|
if os.path.isdir(os.path.join(prevDir,uploadFile)):
|
|
existingFolder=drive.ListFile({'q' : "title = '%s' and '%s' in parents and trashed = false" % (os.path.basename(uploadFile), parent['id'])}).GetList()
|
|
if len(existingFolder) == 0 and (not isInitial or createRoot):
|
|
parent = drive.CreateFile({'title': os.path.basename(uploadFile), 'parents' : [{"kind": "drive#fileLink", 'id' : parent['id']}],
|
|
"mimeType": "application/vnd.google-apps.folder" })
|
|
parent.Upload()
|
|
else:
|
|
if (not isInitial or createRoot) and len(existingFolder) > 0:
|
|
parent=existingFolder[0]
|
|
for f in os.listdir(os.path.join(prevDir,uploadFile)):
|
|
if f not in ignoreFiles:
|
|
copyToDrive(drive, f, True, replaceFiles, ignoreFiles, parent, os.path.join(prevDir,uploadFile))
|
|
else:
|
|
if os.path.basename(uploadFile) not in ignoreFiles:
|
|
existingFiles=drive.ListFile({'q' : "title = '%s' and '%s' in parents and trashed = false" % (os.path.basename(uploadFile), parent['id'])}).GetList()
|
|
if len(existingFiles) > 0:
|
|
driveFile = drive.CreateFile({'title': os.path.basename(uploadFile), 'parents' : [{"kind": "drive#fileLink", 'id' : parent['id']}], })
|
|
else:
|
|
driveFile=existingFiles[0]
|
|
driveFile.SetContentFile(os.path.join(prevDir,uploadFile))
|
|
driveFile.Upload()
|
|
|
|
def watchChange(drive, channel_id, channel_type, channel_address,
|
|
channel_token=None, expiration=None):
|
|
if not drive:
|
|
drive=getDrive()
|
|
if drive.auth.access_token_expired:
|
|
drive.auth.Refresh()
|
|
"""Watch for all changes to a user's Drive.
|
|
Args:
|
|
service: Drive API service instance.
|
|
channel_id: Unique string that identifies this channel.
|
|
channel_type: Type of delivery mechanism used for this channel.
|
|
channel_address: Address where notifications are delivered.
|
|
channel_token: An arbitrary string delivered to the target address with
|
|
each notification delivered over this channel. Optional.
|
|
channel_address: Address where notifications are delivered. Optional.
|
|
Returns:
|
|
The created channel if successful
|
|
Raises:
|
|
apiclient.errors.HttpError: if http request to create channel fails.
|
|
"""
|
|
body = {
|
|
'id': channel_id,
|
|
'type': channel_type,
|
|
'address': channel_address
|
|
}
|
|
if channel_token:
|
|
body['token'] = channel_token
|
|
if expiration:
|
|
body['expiration'] = expiration
|
|
return drive.auth.service.changes().watch(body=body).execute()
|
|
|
|
def watchFile(drive, file_id, channel_id, channel_type, channel_address,
|
|
channel_token=None, expiration=None):
|
|
"""Watch for any changes to a specific file.
|
|
Args:
|
|
service: Drive API service instance.
|
|
file_id: ID of the file to watch.
|
|
channel_id: Unique string that identifies this channel.
|
|
channel_type: Type of delivery mechanism used for this channel.
|
|
channel_address: Address where notifications are delivered.
|
|
channel_token: An arbitrary string delivered to the target address with
|
|
each notification delivered over this channel. Optional.
|
|
channel_address: Address where notifications are delivered. Optional.
|
|
Returns:
|
|
The created channel if successful
|
|
Raises:
|
|
apiclient.errors.HttpError: if http request to create channel fails.
|
|
"""
|
|
if not drive:
|
|
drive=getDrive()
|
|
if drive.auth.access_token_expired:
|
|
drive.auth.Refresh()
|
|
|
|
body = {
|
|
'id': channel_id,
|
|
'type': channel_type,
|
|
'address': channel_address
|
|
}
|
|
if channel_token:
|
|
body['token'] = channel_token
|
|
if expiration:
|
|
body['expiration'] = expiration
|
|
return drive.auth.service.files().watch(fileId=file_id, body=body).execute()
|
|
|
|
def stopChannel(drive, channel_id, resource_id):
|
|
"""Stop watching to a specific channel.
|
|
Args:
|
|
service: Drive API service instance.
|
|
channel_id: ID of the channel to stop.
|
|
resource_id: Resource ID of the channel to stop.
|
|
Raises:
|
|
apiclient.errors.HttpError: if http request to create channel fails.
|
|
"""
|
|
if not drive:
|
|
drive=getDrive()
|
|
if drive.auth.access_token_expired:
|
|
drive.auth.Refresh()
|
|
service=drive.auth.service
|
|
body = {
|
|
'id': channel_id,
|
|
'resourceId': resource_id
|
|
}
|
|
return drive.auth.service.channels().stop(body=body).execute()
|
|
|
|
def getChangeById (drive, change_id):
|
|
if not drive:
|
|
drive=getDrive()
|
|
if drive.auth.access_token_expired:
|
|
drive.auth.Refresh()
|
|
"""Print a single Change resource information.
|
|
|
|
Args:
|
|
service: Drive API service instance.
|
|
change_id: ID of the Change resource to retrieve.
|
|
"""
|
|
try:
|
|
change = drive.auth.service.changes().get(changeId=change_id).execute()
|
|
return change
|
|
except errors.HttpError, error:
|
|
web.app.logger.exception(error)
|
|
return None
|