128 lines
4.8 KiB
Python
128 lines
4.8 KiB
Python
import os;
|
|
import sys;
|
|
import requests;
|
|
import json;
|
|
|
|
from PIL import Image
|
|
from PIL.Image import Exif
|
|
from PIL.ExifTags import TAGS, GPSTAGS
|
|
from io import BytesIO
|
|
|
|
#Get auth info from env variables
|
|
refreshtoken = os.environ["ccprefreshtoken"]
|
|
context = os.environ["ccpcontext"]
|
|
clientid = os.environ["ccpclientid"]
|
|
iam = os.environ["ccpiamurl"]
|
|
|
|
index = int(sys.argv[1]) if len(sys.argv) > 1 else 1
|
|
workers = int(sys.argv[2]) if len(sys.argv) > 2 else 1
|
|
|
|
#Auth related functions
|
|
|
|
# You can use this login data structure while testing
|
|
#logindata = { 'grant_type' : 'password', 'client_id' : clientid, 'username' : '', 'password' : ''}
|
|
logindata = { 'grant_type' : 'refresh_token', 'client_id' : clientid, 'refresh_token' : refreshtoken }
|
|
loginheaders = { "Accept" : "application/json", "Content-Type" : "application/x-www-form-urlencoded", "X-GCube-context" : context}
|
|
|
|
def getToken():
|
|
resp1 = requests.post(iam, data=logindata, headers=loginheaders)
|
|
jwt = resp1.json()
|
|
return jwt["access_token"]
|
|
|
|
token = getToken()
|
|
headers = { "Authorization" : "Bearer " + token}
|
|
|
|
storage_url = "https://workspace-repository.dev.d4science.org/storagehub/workspace"
|
|
|
|
# GET VRE Folder for context
|
|
print("Lookup VRE folder ....")
|
|
vrefolder_url = storage_url + "/vrefolder"
|
|
response = requests.get(vrefolder_url, headers=headers)
|
|
vrefolder_item = response.json()["item"]
|
|
vrefolder_id = vrefolder_item["id"]
|
|
print("Found %(name)s (%(id)s)" % vrefolder_item)
|
|
|
|
# List contents
|
|
print("Looing in VRE folder for a folder named 'pictures' ....")
|
|
vrefolder_list_url = storage_url + "/items/%s/children" % vrefolder_id
|
|
response = requests.get(vrefolder_list_url, headers=headers)
|
|
|
|
# Filter out picture folder
|
|
items = response.json()["itemlist"]
|
|
l = list(filter(lambda x: x["name"] == "pictures", items))
|
|
if(len(l) != 1):
|
|
print("Nothing found")
|
|
exit()
|
|
picturefolder = l[0]
|
|
picturefolder_id = picturefolder["id"]
|
|
print("Found %(name)s (%(id)s)" % picturefolder)
|
|
|
|
# list pictures
|
|
print("Looking for all pictures ...")
|
|
picture_list_url = storage_url + "/items/%s/children" % picturefolder_id
|
|
response = requests.get(picture_list_url, headers=headers)
|
|
items = response.json()["itemlist"]
|
|
items = list(filter(lambda x: x["content"]["mimeType"] in ["image/png", "image/jpg", "image/jpeg"], items))
|
|
print("Found %d pictures." % len(items))
|
|
|
|
# compute start and count indexes
|
|
count = round(len(items) / workers)
|
|
start = (index - 1) * count
|
|
end = start + count
|
|
pictures = items[int(start):int(end)]
|
|
print("Treating %d pictures from %d to %d." % (count, start, end))
|
|
|
|
output = []
|
|
for pic in pictures:
|
|
print("Treating picture %(name)s %(id)s" % pic)
|
|
picture_url = storage_url + "/items/%s/download" % pic["id"]
|
|
response = requests.get(picture_url, headers=headers, stream=True)
|
|
img = Image.open(BytesIO(response.content))
|
|
exif = img.getexif()
|
|
for k,v in TAGS.items():
|
|
if v == "GPSInfo":
|
|
gps_info = exif.get_ifd(k)
|
|
lat = lon = alt = None
|
|
latsign = lonsign = altsign = 1
|
|
for gpsk, gpsv in gps_info.items():
|
|
#print("\t", gpsk, GPSTAGS.get(gpsk), gpsv)
|
|
if GPSTAGS.get(gpsk) == "GPSLatitude": lat = float(gpsv[0]) + float(gpsv[1]) / 60 + float(gpsv[2]) / 3600
|
|
if GPSTAGS.get(gpsk) == "GPSLatitudeRef": latsign = -1 if gpsv == 'S' else 1
|
|
if GPSTAGS.get(gpsk) == "GPSLongitude": lon = float(gpsv[0]) + float(gpsv[1]) / 60 + float(gpsv[2]) / 3600
|
|
if GPSTAGS.get(gpsk) == "GPSLongitudeRef": lonsign = -1 if gpsv == 'W' else 1
|
|
if GPSTAGS.get(gpsk) == "GPSAltitude": alt = float(gpsv)
|
|
if GPSTAGS.get(gpsk) == "GPSAltitudeRef": altsign = -1 if gpsv == -1 else 1
|
|
if lat != None and lon != None and alt != None:
|
|
out = {'id' : pic['id'], 'name': pic['name'], 'url' : picture_url, 'lat' : lat * latsign, 'lon': lon * lonsign, 'alt' : alt * altsign}
|
|
print("GPS found: %(lon)f,%(lat)f altitude %(alt)f" % out)
|
|
output.append(out)
|
|
else:
|
|
pass
|
|
|
|
# refresh token
|
|
token = getToken()
|
|
headers = { "Authorization" : "Bearer " + token}
|
|
|
|
# build kml entry and upload
|
|
for o in output:
|
|
kmloutput = '''
|
|
<kml xmlns="http://www.opengis.net/kml/2.2">
|
|
<Document id="%(id)s">
|
|
<Placemark>
|
|
<name>%(name)s</name>
|
|
<Point>
|
|
<coordinates>%(lon)f,%(lat)f,%(alt)f</coordinates>
|
|
</Point>
|
|
</Placemark>
|
|
</Document>
|
|
</kml>''' % o
|
|
# Create index file
|
|
files = {
|
|
'name': (None, '%(id)s.kml' % o),
|
|
'description': (None, 'KML indexing of item %(name)s' % o),
|
|
'file': ('%(id)s.kml' % o, kmloutput)
|
|
}
|
|
print("Uploading kml entry as %s " % storage_url + "/items/" + picturefolder_id + "/create/FILE")
|
|
indexentry = requests.post(storage_url + "/items/" + picturefolder_id + "/create/FILE", files=files, headers=headers)
|
|
print(indexentry)
|