remonet/mapillary_download
1
0
Fork 0
forked from tykayn/mapillary_download
Code Pull Requests Activity

Switching to pyexiv2

Browse Source
This commit is contained in:
Stefal 2023-09-12 14:31:10 +02:00
parent fc7e82350d
commit 6989b5e0d9
17 changed files with 80 additions and 3796 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,4 +1,2 @@
data/*
lib/__pycache__/
lib/test
__pycache__/

73
download.py
View File

@ -6,11 +6,13 @@ import argparse
from datetime import datetime
from lib.exif_write import ExifEdit
import writer
from model import PictureType
def parse_args(argv =None):
parser = argparse.ArgumentParser()
parser.add_argument('--sequence_id', type=str, help='The mapillary sequence id to download')
parser.add_argument('--access_token', type=str, help='Your mapillary access token')
parser.add_argument('--image_count', type=int, default=None, help='How many images you want to download')
global args
args = parser.parse_args(argv)
@ -22,56 +24,23 @@ def background(f):
#@background
def download(url, fn, metadata=None):
r = requests.get(url, stream=True)
with open(str(fn), "wb") as f:
f.write(r.content)
write_exif(fn, metadata)
r = requests.get(url, stream=True)
image = write_exif(r.content, metadata)
with open(str(fn), "wb") as f:
f.write(image)
def write_exif(filename, img_metadata):
def write_exif(picture, img_metadata):
'''
Write exif metadata
'''
#{'thumb_original_url': 'https://scontent-cdg4-2.xx.fbcdn.net/m1/v/t6/An9Zy2SrH9vXJIF01QkBODyUbg7XSKfwL48UwHyvihSwvECGjVbG0vSw9uhxe2-Dq-k2eUcigb83buO6zo-7eVbykfp5aQIe1kgd-MJr66nU_H-o_mwBLZXgVbj5I_5WX-C9c6FxJruHkV962F228O0?ccb=10-5&oh=00_AfDOKD869DxL-4ZNCbVo8Rn29vsc0JyjMAU2ctx4aAFVMQ&oe=65256C25&_nc_sid=201bca',
# 'captured_at': 1603459736644, 'geometry': {'type': 'Point', 'coordinates': [2.5174596904057, 48.777089857534]}, 'id': '485924785946693'}
""" lat = data['geometry']['coordinates'][1]
long = data['geometry']['coordinates'][0]
altitude = data['altitude']
bearing = data['compass_angle']
timestamp=datetime.utcfromtimestamp(int(data['captured_at'])/1000)
"""
metadata = ExifEdit(filename)
#metadata.read()
try:
# add to exif
#metadata["Exif.GPSInfo.GPSLatitude"] = exiv_lat
#metadata["Exif.GPSInfo.GPSLatitudeRef"] = coordinates[3]
#metadata["Exif.GPSInfo.GPSLongitude"] = exiv_lon
#metadata["Exif.GPSInfo.GPSLongitudeRef"] = coordinates[7]
#metadata["Exif.GPSInfo.GPSMapDatum"] = "WGS-84"
#metadata["Exif.GPSInfo.GPSVersionID"] = '2 0 0 0'
#metadata["Exif.GPSInfo.GPSImgDirection"] = exiv_bearing
#metadata["Exif.GPSInfo.GPSImgDirectionRef"] = "T"
metadata.add_lat_lon(img_metadata.latitude, img_metadata.longitude)
metadata.add_altitude(img_metadata.altitude)
metadata.add_date_time_original(img_metadata.capture_time)
metadata.add_direction(img_metadata.direction)
#if data['camera_type'] == 'spherical'
metadata.write()
print("Added geodata to: {0}".format(filename))
except ValueError as e:
print("Skipping {0}: {1}".format(filename, e))
if img_metadata.picture_type == "equirectangular" :
print('Pano detected')
import pyexiv2
img = pyexiv2.Image(filename)
xmp_pano = {'Xmp.GPano.ProjectionType' : 'Equirectangular'}
img.modify_xmp(xmp_pano)
img.close()
picture = writer.writePictureMetadata(picture, img_metadata)
picture = writer.add_altitude(picture, img_metadata)
picture = writer.add_direction(picture, img_metadata)
return picture
if __name__ == '__main__':
parse_args()
@ -101,12 +70,12 @@ if __name__ == '__main__':
data = r.json()
image_ids = data['data']
img_num = len(image_ids)
img_num = args.image_count if args.image_count is not None else len(image_ids)
urls = []
print(img_num)
print('getting urls')
#for x in range(0, img_num):
for x in range(0, 5):
for x in range(0, img_num):
#for x in range(0, 5):
image_id = image_ids[x]['id']
req_url = 'https://graph.mapillary.com/{}?fields=thumb_original_url,altitude,camera_type,captured_at,compass_angle,geometry,exif_orientation'.format(image_id)
r = requests.get(req_url, headers=header)
@ -120,11 +89,11 @@ if __name__ == '__main__':
date_time_image_filename = datetime.utcfromtimestamp(int(url['captured_at'])/1000).strftime('%Y-%m-%d_%HH%Mmn%S.%f')
path = 'data/{}/{}.jpg'.format(sequence_id, date_time_image_filename)
img_metadata = writer.PictureMetadata(
capture_time = datetime.utcfromtimestamp(int(url['captured_at'])/1000) ,
longitude = url['geometry']['coordinates'][0] ,
latitude = url['geometry']['coordinates'][1] ,
picture_type = "equirectangular" if url['camera_type'] == 'spherical' else None ,
direction = url['compass_angle'] ,
altitude = url['altitude']
capture_time = datetime.utcfromtimestamp(int(url['captured_at'])/1000),
longitude = url['geometry']['coordinates'][0],
latitude = url['geometry']['coordinates'][1],
picture_type = PictureType("equirectangular") if url['camera_type'] == 'spherical' else None,
direction = url['compass_angle'],
altitude = url['altitude'],
)
download(url['thumb_original_url'],path, img_metadata)

17
lib/__init__.py
View File

@ -1,17 +0,0 @@
#from .geo import *
#from .exif_aux import *
#from .exif_read import *
#from .exif_write import *
#from .gps_parser import *
#from .gpmf import *
#import geo
#import exif_aux
#import exif_read
#import exif_write
#import gps_parser
#import gpmf
VERSION = "0.0.2"

385
lib/exif.py
View File

@ -1,385 +0,0 @@
#!/usr/bin/env python
import os
import sys
import exifread
import datetime
from lib.geo import normalize_bearing
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
def eval_frac(value):
return float(value.num) / float(value.den)
def exif_gps_fields():
'''
GPS fields in EXIF
'''
return [
["GPS GPSLongitude", "EXIF GPS GPSLongitude"],
["GPS GPSLatitude", "EXIF GPS GPSLatitude"]
]
def exif_datetime_fields():
'''
Date time fields in EXIF
'''
return [["EXIF DateTimeOriginal",
"Image DateTimeOriginal",
"EXIF DateTimeDigitized",
"Image DateTimeDigitized",
"EXIF DateTime"
"Image DateTime",
"GPS GPSDate",
"EXIF GPS GPSDate",
"EXIF DateTimeModified"]]
def format_time(time_string):
'''
Format time string with invalid time elements in hours/minutes/seconds
Format for the timestring needs to be "%Y_%m_%d_%H_%M_%S"
e.g. 2014_03_31_24_10_11 => 2014_04_01_00_10_11
'''
data = time_string.split("_")
hours, minutes, seconds = int(data[3]), int(data[4]), int(data[5])
date = datetime.datetime.strptime("_".join(data[:3]), "%Y_%m_%d")
date_time = date + datetime.timedelta(hours=hours, minutes=minutes, seconds=seconds)
return date_time
def gps_to_decimal(values, reference):
sign = 1 if reference in 'NE' else -1
degrees = eval_frac(values[0])
minutes = eval_frac(values[1])
seconds = eval_frac(values[2])
return sign * (degrees + minutes / 60 + seconds / 3600)
def get_float_tag(tags, key):
if key in tags:
return float(tags[key].values[0])
else:
return None
def get_frac_tag(tags, key):
if key in tags:
return eval_frac(tags[key].values[0])
else:
return None
def extract_exif_from_file(fileobj):
if isinstance(fileobj, (str, unicode)):
with open(fileobj) as f:
exif_data = EXIF(f)
else:
exif_data = EXIF(fileobj)
d = exif_data.extract_exif()
return d
def required_fields():
return exif_gps_fields() + exif_datetime_fields()
def verify_exif(filename):
'''
Check that image file has the required EXIF fields.
Incompatible files will be ignored server side.
'''
# required tags in IFD name convention
required_exif = required_fields()
exif = EXIF(filename)
required_exif_exist = exif.fields_exist(required_exif)
return required_exif_exist
def verify_mapillary_tag(filename):
'''
Check that image file has the required Mapillary tag
'''
return EXIF(filename).mapillary_tag_exists()
def is_image(filename):
return filename.lower().endswith(('jpg', 'jpeg', 'png', 'tif', 'tiff', 'pgm', 'pnm', 'gif'))
class EXIF:
'''
EXIF class for reading exif from an image
'''
def __init__(self, filename, details=False):
'''
Initialize EXIF object with FILE as filename or fileobj
'''
self.filename = filename
if type(filename) == str:
with open(filename, 'rb') as fileobj:
self.tags = exifread.process_file(fileobj, details=details)
else:
self.tags = exifread.process_file(filename, details=details)
def _extract_alternative_fields(self, fields, default=None, field_type=float):
'''
Extract a value for a list of ordered fields.
Return the value of the first existed field in the list
'''
for field in fields:
if field in self.tags:
if field_type is float:
value = eval_frac(self.tags[field].values[0])
if field_type is str:
value = str(self.tags[field].values)
if field_type is int:
value = int(self.tags[field].values[0])
return value, field
return default, None
def exif_name(self):
'''
Name of file in the form {lat}_{lon}_{ca}_{datetime}_{filename}
'''
lon, lat = self.extract_lon_lat()
ca = self.extract_direction()
if ca is None:
ca = 0
ca = int(ca)
date_time = self.extract_capture_time()
date_time = date_time.strftime("%Y-%m-%d-%H-%M-%S-%f")
date_time = date_time[:-3]
filename = '{}_{}_{}_{}_{}'.format(lat, lon, ca, date_time, os.path.basename(self.filename))
return filename
def extract_altitude(self):
'''
Extract altitude
'''
fields = ['GPS GPSAltitude', 'EXIF GPS GPSAltitude']
altitude, _ = self._extract_alternative_fields(fields, 0, float)
return altitude
def extract_capture_time(self):
'''
Extract capture time from EXIF
return a datetime object
TODO: handle GPS DateTime
'''
time_string = exif_datetime_fields()[0]
capture_time, time_field = self._extract_alternative_fields(time_string, 0, str)
# if "GPSDate" in time_field:
# return self.extract_gps_time()
if capture_time is 0:
# try interpret the filename
try:
capture_time = datetime.datetime.strptime(os.path.basename(self.filename)[:-4]+'000', '%Y_%m_%d_%H_%M_%S_%f')
except:
pass
else:
capture_time = capture_time.replace(" ", "_")
capture_time = capture_time.replace(":", "_")
capture_time = "_".join(["{0:02d}".format(int(ts)) for ts in capture_time.split("_") if ts.isdigit()])
capture_time = format_time(capture_time)
sub_sec = self.extract_subsec()
capture_time = capture_time + datetime.timedelta(seconds=float(sub_sec)/10**len(str(sub_sec)))
return capture_time
def extract_direction(self):
'''
Extract image direction (i.e. compass, heading, bearing)
'''
fields = ['GPS GPSImgDirection',
'EXIF GPS GPSImgDirection',
'GPS GPSTrack',
'EXIF GPS GPSTrack']
direction, _ = self._extract_alternative_fields(fields)
if direction is not None:
direction = normalize_bearing(direction, check_hex=True)
return direction
def extract_dop(self):
'''
Extract dilution of precision
'''
fields = ['GPS GPSDOP', 'EXIF GPS GPSDOP']
dop, _ = self._extract_alternative_fields(fields)
return dop
def extract_geo(self):
'''
Extract geo-related information from exif
'''
altitude = self.extract_altitude()
dop = self.extract_dop()
lon, lat = self.extract_lon_lat()
d = {}
if lon is not None and lat is not None:
d['latitude'] = lat
d['longitude'] = lon
if altitude is not None:
d['altitude'] = altitude
if dop is not None:
d['dop'] = dop
return d
def extract_gps_time(self):
'''
Extract timestamp from GPS field.
'''
gps_date_field = "GPS GPSDate"
gps_time_field = "GPS GPSTimeStamp"
gps_time = 0
if gps_date_field in self.tags and gps_time_field in self.tags:
date = str(self.tags[gps_date_field].values).split(":")
t = self.tags[gps_time_field]
gps_time = datetime.datetime(
year=int(date[0]),
month=int(date[1]),
day=int(date[2]),
hour=int(eval_frac(t.values[0])),
minute=int(eval_frac(t.values[1])),
second=int(eval_frac(t.values[2])),
)
microseconds = datetime.timedelta(microseconds=int( (eval_frac(t.values[2])%1) *1e6))
gps_time += microseconds
return gps_time
def extract_exif(self):
'''
Extract a list of exif infos
'''
width, height = self.extract_image_size()
make, model = self.extract_make(), self.extract_model()
orientation = self.extract_orientation()
geo = self.extract_geo()
capture = self.extract_capture_time()
direction = self.extract_direction()
d = {
'width': width,
'height': height,
'orientation': orientation,
'direction': direction,
'make': make,
'model': model,
'capture_time': capture
}
d['gps'] = geo
return d
def extract_image_size(self):
'''
Extract image height and width
'''
width, _ = self._extract_alternative_fields(['Image ImageWidth', 'EXIF ExifImageWidth'], -1, int)
height, _ = self._extract_alternative_fields(['Image ImageLength', 'EXIF ExifImageLength'], -1, int)
return width, height
def extract_image_description(self):
'''
Extract image description
'''
description, _ = self._extract_alternative_fields(['Image ImageDescription'], "{}", str)
return description
def extract_lon_lat(self):
if 'GPS GPSLatitude' in self.tags and 'GPS GPSLatitude' in self.tags:
lat = gps_to_decimal(self.tags['GPS GPSLatitude'].values,
self.tags['GPS GPSLatitudeRef'].values)
lon = gps_to_decimal(self.tags['GPS GPSLongitude'].values,
self.tags['GPS GPSLongitudeRef'].values)
elif 'EXIF GPS GPSLatitude' in self.tags and 'EXIF GPS GPSLatitude' in self.tags:
lat = gps_to_decimal(self.tags['EXIF GPS GPSLatitude'].values,
self.tags['EXIF GPS GPSLatitudeRef'].values)
lon = gps_to_decimal(self.tags['EXIF GPS GPSLongitude'].values,
self.tags['EXIF GPS GPSLongitudeRef'].values)
else:
lon, lat = None, None
return lon, lat
def extract_make(self):
'''
Extract camera make
'''
fields = ['EXIF LensMake', 'Image Make']
make, _ = self._extract_alternative_fields(fields, default='none', field_type=str)
return make
def extract_model(self):
'''
Extract camera model
'''
fields = ['EXIF LensModel', 'Image Model']
model, _ = self._extract_alternative_fields(fields, default='none', field_type=str)
return model
def extract_orientation(self):
'''
Extract image orientation
'''
fields = ['Image Orientation']
orientation, _ = self._extract_alternative_fields(fields, default=1, field_type=int)
if orientation not in [1, 3, 6, 8]:
return 1
return orientation
def extract_subsec(self):
'''
Extract microseconds
'''
fields = [
'Image SubSecTimeOriginal',
'EXIF SubSecTimeOriginal',
'Image SubSecTimeDigitized',
'EXIF SubSecTimeDigitized',
'Image SubSecTime',
'EXIF SubSecTime'
]
sub_sec, _ = self._extract_alternative_fields(fields, default=0, field_type=str)
sub_sec = int(sub_sec)
return sub_sec
def fields_exist(self, fields):
'''
Check existence of a list fields in exif
'''
for rexif in fields:
vflag = False
for subrexif in rexif:
if subrexif in self.tags:
vflag = True
if not vflag:
print("Missing required EXIF tag: {0} for image {1}".format(rexif[0], self.filename))
return False
return True
def mapillary_tag_exists(self):
'''
Check existence of Mapillary tag
'''
description_tag = "Image ImageDescription"
if description_tag in self.tags:
if "MAPSequenceUUID" in self.tags[description_tag].values:
return True
return False

227
lib/exif_pil.py
View File

@ -1,227 +0,0 @@
import datetime
import struct # Only to catch struct.error due to error in PIL / Pillow.
from PIL import Image
from PIL.ExifTags import TAGS, GPSTAGS
# Original: https://gist.github.com/erans/983821
# License: MIT
# Credits: https://gist.github.com/erans
class ExifException(Exception):
def __init__(self, message):
self._message = message
def __str__(self):
return self._message
class PILExifReader:
def __init__(self, filepath):
self._filepath = filepath
image = Image.open(filepath)
self._exif = self.get_exif_data(image)
image.close()
def get_exif_data(self, image):
"""Returns a dictionary from the exif data of an PIL Image
item. Also converts the GPS Tags"""
exif_data = {}
try:
info = image._getexif()
except OverflowError, e:
if e.message == "cannot fit 'long' into an index-sized integer":
# Error in PIL when exif data is corrupt.
return None
else:
raise e
except struct.error as e:
if e.message == "unpack requires a string argument of length 2":
# Error in PIL when exif data is corrupt.
return None
else:
raise e
if info:
for tag, value in info.items():
decoded = TAGS.get(tag, tag)
if decoded == "GPSInfo":
gps_data = {}
for t in value:
sub_decoded = GPSTAGS.get(t, t)
gps_data[sub_decoded] = value[t]
exif_data[decoded] = gps_data
else:
exif_data[decoded] = value
return exif_data
def read_capture_time(self):
time_tag = "DateTimeOriginal"
# read and format capture time
if self._exif == None:
print "Exif is none."
if time_tag in self._exif:
capture_time = self._exif[time_tag]
capture_time = capture_time.replace(" ","_")
capture_time = capture_time.replace(":","_")
else:
print "No time tag in "+self._filepath
capture_time = 0
# return as datetime object
return datetime.datetime.strptime(capture_time, '%Y_%m_%d_%H_%M_%S')
def _get_if_exist(self, data, key):
if key in data:
return data[key]
else:
return None
def _convert_to_degress(self, value):
"""Helper function to convert the GPS coordinates stored in
the EXIF to degrees in float format."""
d0 = value[0][0]
d1 = value[0][1]
d = float(d0) / float(d1)
m0 = value[1][0]
m1 = value[1][1]
m = float(m0) / float(m1)
s0 = value[2][0]
s1 = value[2][1]
s = float(s0) / float(s1)
return d + (m / 60.0) + (s / 3600.0)
def get_lat_lon(self):
"""Returns the latitude and longitude, if available, from the
provided exif_data (obtained through get_exif_data above)."""
lat = None
lon = None
gps_info = self.get_gps_info()
if gps_info is None:
return None
gps_latitude = self._get_if_exist(gps_info, "GPSLatitude")
gps_latitude_ref = self._get_if_exist(gps_info, 'GPSLatitudeRef')
gps_longitude = self._get_if_exist(gps_info, 'GPSLongitude')
gps_longitude_ref = self._get_if_exist(gps_info, 'GPSLongitudeRef')
if (gps_latitude and gps_latitude_ref
and gps_longitude and gps_longitude_ref):
lat = self._convert_to_degress(gps_latitude)
if gps_latitude_ref != "N":
lat = 0 - lat
lon = self._convert_to_degress(gps_longitude)
if gps_longitude_ref != "E":
lon = 0 - lon
if isinstance(lat, float) and isinstance(lon, float):
return lat, lon
else:
return None
def calc_tuple(self, tup):
if tup is None or len(tup) != 2 or tup[1] == 0:
return None
return int(tup[0]) / int(tup[1])
def get_gps_info(self):
if self._exif is None or not "GPSInfo" in self._exif:
return None
else:
return self._exif["GPSInfo"]
def get_rotation(self):
"""Returns the direction of the GPS receiver in degrees."""
gps_info = self.get_gps_info()
if gps_info is None:
return None
for tag in ('GPSImgDirection', 'GPSTrack'):
gps_direction = self._get_if_exist(gps_info, tag)
direction = self.calc_tuple(gps_direction)
if direction == None:
continue
else:
return direction
return None
def get_speed(self):
"""Returns the GPS speed in km/h or None if it does not exists."""
gps_info = self.get_gps_info()
if gps_info is None:
return None
if not "GPSSpeed" in gps_info or not "GPSSpeedRef" in gps_info:
return None
speed_frac = gps_info["GPSSpeed"]
speed_ref = gps_info["GPSSpeedRef"]
speed = self.calc_tuple(speed_frac)
if speed is None or speed_ref is None:
return None
speed_ref = speed_ref.lower()
if speed_ref == "k":
pass # km/h - we are happy.
elif speed_ref == "m":
#Miles pr. hour => km/h
speed *= 1.609344
elif speed_ref == "n":
# Knots => km/h
speed *= 1.852
else:
print "Warning: Unknown format for GPS speed '%s' in '%s'." % (
speed_ref, self._filepath)
print "Please file a bug and attache the image."
return None
return speed
def is_ok_num(self, val, minVal, maxVal):
try:
num = int(val)
except ValueError:
return False
if num < minVal or num > maxVal:
return False
return True
def get_time(self):
# Example data
# GPSTimeStamp': ((9, 1), (14, 1), (9000, 1000))
# 'GPSDateStamp': u'2015:05:17'
gps_info = self.get_gps_info()
if gps_info is None:
return None
if not 'GPSTimeStamp' in gps_info or not 'GPSDateStamp' in gps_info:
return None
timestamp = gps_info['GPSTimeStamp']
datestamp = gps_info['GPSDateStamp']
if len(timestamp) != 3:
raise ExifException("Timestamp does not have length 3: %s" %
len(timestamp))
(timeH, timeM, timeS) = timestamp
h = self.calc_tuple(timeH)
m = self.calc_tuple(timeM)
s = self.calc_tuple(timeS)
if None in (h, m, s):
raise ExifException(
"Hour, minute or second is not valid: '%s':'%s':'%s'." %
(timeH, timeM, timeS))
if datestamp.count(':') != 2:
raise ExifException("Datestamp does not contain 2 colons: '%s'" %
datestamp)
(y, mon, d) = [int(str) for str in datestamp.split(':')]
if not self.is_ok_num(y, 1970, 2100) or not self.is_ok_num(
mon, 1, 12) or not self.is_ok_num(d, 1, 31):
raise ExifException(
"Date parsed from the following is not OK: '%s'" % datestamp)
return datetime.datetime(y, mon, d, h, m, s)

370
lib/exif_read.py
View File

@ -1,370 +0,0 @@
# coding: utf8
#!/usr/bin/env python
#source is exif_read.py from mapillary_tools :
#https://github.com/mapillary/mapillary_tools/blob/master/mapillary_tools/exif_read.py
import os
import sys
import exifread
import datetime
from geo import normalize_bearing
import uuid
sys.path.insert(0, os.path.abspath(
os.path.join(os.path.dirname(__file__), "..")))
#import jsonfrom
def eval_frac(value):
if value.den == 0:
return -1.0
return float(value.num) / float(value.den)
def format_time(time_string):
'''
Format time string with invalid time elements in hours/minutes/seconds
Format for the timestring needs to be "%Y_%m_%d_%H_%M_%S"
e.g. 2014_03_31_24_10_11 => 2014_04_01_00_10_11
'''
subseconds = False
data = time_string.split("_")
hours, minutes, seconds = int(data[3]), int(data[4]), int(data[5])
date = datetime.datetime.strptime("_".join(data[:3]), "%Y_%m_%d")
subsec = 0.0
if len(data) == 7:
if float(data[6]) != 0:
subsec = float(data[6]) / 10**len(data[6])
subseconds = True
date_time = date + \
datetime.timedelta(hours=hours, minutes=minutes,
seconds=seconds + subsec)
return date_time, subseconds
def gps_to_decimal(values, reference):
sign = 1 if reference in 'NE' else -1
degrees = eval_frac(values[0])
minutes = eval_frac(values[1])
seconds = eval_frac(values[2])
return sign * (degrees + minutes / 60 + seconds / 3600)
def exif_datetime_fields():
'''
Date time fields in EXIF
'''
return [["EXIF DateTimeOriginal",
"Image DateTimeOriginal",
"EXIF DateTimeDigitized",
"Image DateTimeDigitized",
"EXIF DateTime",
"Image DateTime",
"GPS GPSDate",
"EXIF GPS GPSDate",
"EXIF DateTimeModified"]]
def exif_gps_date_fields():
'''
Date fields in EXIF GPS
'''
return [["GPS GPSDate",
"EXIF GPS GPSDate"]]
class ExifRead:
'''
EXIF class for reading exif from an image
'''
def __init__(self, filename, details=False):
'''
Initialize EXIF object with FILE as filename or fileobj
'''
self.filename = filename
if type(filename) == str:
with open(filename, 'rb') as fileobj:
self.tags = exifread.process_file(fileobj, details=details)
else:
self.tags = exifread.process_file(filename, details=details)
def _extract_alternative_fields(self, fields, default=None, field_type=float):
'''
Extract a value for a list of ordered fields.
Return the value of the first existed field in the list
'''
for field in fields:
if field in self.tags:
if field_type is float:
value = eval_frac(self.tags[field].values[0])
if field_type is str:
value = str(self.tags[field].values)
if field_type is int:
value = int(self.tags[field].values[0])
return value, field
return default, None
def exif_name(self):
'''
Name of file in the form {lat}_{lon}_{ca}_{datetime}_{filename}_{hash}
'''
mapillary_description = json.loads(self.extract_image_description())
lat = None
lon = None
ca = None
date_time = None
if "MAPLatitude" in mapillary_description:
lat = mapillary_description["MAPLatitude"]
if "MAPLongitude" in mapillary_description:
lon = mapillary_description["MAPLongitude"]
if "MAPCompassHeading" in mapillary_description:
if 'TrueHeading' in mapillary_description["MAPCompassHeading"]:
ca = mapillary_description["MAPCompassHeading"]['TrueHeading']
if "MAPCaptureTime" in mapillary_description:
date_time = datetime.datetime.strptime(
mapillary_description["MAPCaptureTime"], "%Y_%m_%d_%H_%M_%S_%f").strftime("%Y-%m-%d-%H-%M-%S-%f")[:-3]
filename = '{}_{}_{}_{}_{}'.format(
lat, lon, ca, date_time, uuid.uuid4())
return filename
def extract_image_history(self):
field = ['Image Tag 0x9213']
user_comment, _ = self._extract_alternative_fields(field, '{}', str)
return user_comment
def extract_altitude(self):
'''
Extract altitude
'''
fields = ['GPS GPSAltitude', 'EXIF GPS GPSAltitude']
altitude, _ = self._extract_alternative_fields(fields, 0, float)
return altitude
def extract_capture_time(self):
'''
Extract capture time from EXIF
return a datetime object
TODO: handle GPS DateTime
'''
time_string = exif_datetime_fields()[0]
capture_time, time_field = self._extract_alternative_fields(
time_string, 0, str)
if time_field in exif_gps_date_fields()[0]:
capture_time = self.extract_gps_time()
return capture_time
if capture_time is 0:
# try interpret the filename
try:
capture_time = datetime.datetime.strptime(os.path.basename(
self.filename)[:-4] + '000', '%Y_%m_%d_%H_%M_%S_%f')
except:
return None
else:
capture_time = capture_time.replace(" ", "_")
capture_time = capture_time.replace(":", "_")
capture_time = capture_time.replace(".", "_")
capture_time = capture_time.replace("-", "_")
capture_time = "_".join(
[ts for ts in capture_time.split("_") if ts.isdigit()])
capture_time, subseconds = format_time(capture_time)
sub_sec = "0"
if not subseconds:
sub_sec = self.extract_subsec()
capture_time = capture_time + \
datetime.timedelta(seconds=float("0." + sub_sec))
return capture_time
def extract_direction(self):
'''
Extract image direction (i.e. compass, heading, bearing)
'''
fields = ['GPS GPSImgDirection',
'EXIF GPS GPSImgDirection',
'GPS GPSTrack',
'EXIF GPS GPSTrack']
direction, _ = self._extract_alternative_fields(fields)
if direction is not None:
direction = normalize_bearing(direction, check_hex=True)
return direction
def extract_dop(self):
'''
Extract dilution of precision
'''
fields = ['GPS GPSDOP', 'EXIF GPS GPSDOP']
dop, _ = self._extract_alternative_fields(fields)
return dop
def extract_geo(self):
'''
Extract geo-related information from exif
'''
altitude = self.extract_altitude()
dop = self.extract_dop()
lon, lat = self.extract_lon_lat()
d = {}
if lon is not None and lat is not None:
d['latitude'] = lat
d['longitude'] = lon
if altitude is not None:
d['altitude'] = altitude
if dop is not None:
d['dop'] = dop
return d
def extract_gps_time(self):
'''
Extract timestamp from GPS field.
'''
gps_date_field = "GPS GPSDate"
gps_time_field = "GPS GPSTimeStamp"
gps_time = 0
if gps_date_field in self.tags and gps_time_field in self.tags:
date = str(self.tags[gps_date_field].values).split(":")
t = self.tags[gps_time_field]
gps_time = datetime.datetime(
year=int(date[0]),
month=int(date[1]),
day=int(date[2]),
hour=int(eval_frac(t.values[0])),
minute=int(eval_frac(t.values[1])),
second=int(eval_frac(t.values[2])),
)
microseconds = datetime.timedelta(
microseconds=int((eval_frac(t.values[2]) % 1) * 1e6))
gps_time += microseconds
return gps_time
def extract_exif(self):
'''
Extract a list of exif infos
'''
width, height = self.extract_image_size()
make, model = self.extract_make(), self.extract_model()
orientation = self.extract_orientation()
geo = self.extract_geo()
capture = self.extract_capture_time()
direction = self.extract_direction()
d = {
'width': width,
'height': height,
'orientation': orientation,
'direction': direction,
'make': make,
'model': model,
'capture_time': capture
}
d['gps'] = geo
return d
def extract_image_size(self):
'''
Extract image height and width
'''
width, _ = self._extract_alternative_fields(
['Image ImageWidth', 'EXIF ExifImageWidth'], -1, int)
height, _ = self._extract_alternative_fields(
['Image ImageLength', 'EXIF ExifImageLength'], -1, int)
return width, height
def extract_image_description(self):
'''
Extract image description
'''
description, _ = self._extract_alternative_fields(
['Image ImageDescription'], "{}", str)
return description
def extract_lon_lat(self):
if 'GPS GPSLatitude' in self.tags and 'GPS GPSLatitude' in self.tags:
lat = gps_to_decimal(self.tags['GPS GPSLatitude'].values,
self.tags['GPS GPSLatitudeRef'].values)
lon = gps_to_decimal(self.tags['GPS GPSLongitude'].values,
self.tags['GPS GPSLongitudeRef'].values)
elif 'EXIF GPS GPSLatitude' in self.tags and 'EXIF GPS GPSLatitude' in self.tags:
lat = gps_to_decimal(self.tags['EXIF GPS GPSLatitude'].values,
self.tags['EXIF GPS GPSLatitudeRef'].values)
lon = gps_to_decimal(self.tags['EXIF GPS GPSLongitude'].values,
self.tags['EXIF GPS GPSLongitudeRef'].values)
else:
lon, lat = None, None
return lon, lat
def extract_make(self):
'''
Extract camera make
'''
fields = ['EXIF LensMake', 'Image Make']
make, _ = self._extract_alternative_fields(
fields, default='none', field_type=str)
return make
def extract_model(self):
'''
Extract camera model
'''
fields = ['EXIF LensModel', 'Image Model']
model, _ = self._extract_alternative_fields(
fields, default='none', field_type=str)
return model
def extract_orientation(self):
'''
Extract image orientation
'''
fields = ['Image Orientation']
orientation, _ = self._extract_alternative_fields(
fields, default=1, field_type=int)
if orientation not in range(1, 9):
return 1
return orientation
def extract_subsec(self):
'''
Extract microseconds
'''
fields = [
'Image SubSecTimeOriginal',
'EXIF SubSecTimeOriginal',
'Image SubSecTimeDigitized',
'EXIF SubSecTimeDigitized',
'Image SubSecTime',
'EXIF SubSecTime'
]
sub_sec, _ = self._extract_alternative_fields(
fields, default='', field_type=str)
return sub_sec.strip()
def fields_exist(self, fields):
'''
Check existence of a list fields in exif
'''
for rexif in fields:
vflag = False
for subrexif in rexif:
if subrexif in self.tags:
vflag = True
if not vflag:
print("Missing required EXIF tag: {0} for image {1}".format(
rexif[0], self.filename))
return False
return True
def mapillary_tag_exists(self):
'''
Check existence of required Mapillary tags
'''
description_tag = "Image ImageDescription"
if description_tag not in self.tags:
return False
for requirement in ["MAPSequenceUUID", "MAPSettingsUserKey", "MAPCaptureTime", "MAPLongitude", "MAPLatitude"]:
if requirement not in self.tags[description_tag].values or json.loads(self.tags[description_tag].values)[requirement] in ["", None, " "]:
return False
return True

122
lib/exif_write.py
View File

@ -1,122 +0,0 @@
import sys
import json
import piexif
from . geo import decimal_to_dms
#from .error import print_error
class ExifEdit(object):
def __init__(self, filename):
"""Initialize the object"""
self._filename = filename
self._ef = None
try:
self._ef = piexif.load(filename)
except IOError:
etype, value, traceback = sys.exc_info()
print >> sys.stderr, "Error opening file:", value
except ValueError:
etype, value, traceback = sys.exc_info()
print >> sys.stderr, "Error opening file:", value
def add_image_description(self, dict):
"""Add a dict to image description."""
if self._ef is not None:
self._ef['0th'][piexif.ImageIFD.ImageDescription] = json.dumps(
dict)
def add_orientation(self, orientation):
"""Add image orientation to image."""
if not orientation in range(1, 9):
print(
"Error value for orientation, value must be in range(1,9), setting to default 1")
self._ef['0th'][piexif.ImageIFD.Orientation] = 1
else:
self._ef['0th'][piexif.ImageIFD.Orientation] = orientation
def add_date_time_original(self, date_time):
"""Add date time original."""
try:
DateTimeOriginal = date_time.strftime('%Y:%m:%d %H:%M:%S')
self._ef['Exif'][piexif.ExifIFD.DateTimeOriginal] = DateTimeOriginal
except Exception as e:
print("Error writing DateTimeOriginal, due to " + str(e))
if date_time.microsecond != 0:
self.add_subsectimeoriginal(date_time.microsecond)
def add_subsectimeoriginal(self, subsec_value):
"""Add subsecond value in the subsectimeoriginal exif tag"""
try:
subsec = str(subsec_value).zfill(6)
self._ef['Exif'][piexif.ExifIFD.SubSecTimeOriginal] = subsec
except Exception as e:
print("Error writing SubSecTimeOriginal, due to " + str(e))
def add_lat_lon(self, lat, lon, precision=1e7):
"""Add lat, lon to gps (lat, lon in float)."""
self._ef["GPS"][piexif.GPSIFD.GPSLatitudeRef] = "N" if lat > 0 else "S"
self._ef["GPS"][piexif.GPSIFD.GPSLongitudeRef] = "E" if lon > 0 else "W"
self._ef["GPS"][piexif.GPSIFD.GPSLongitude] = decimal_to_dms(
abs(lon), int(precision))
self._ef["GPS"][piexif.GPSIFD.GPSLatitude] = decimal_to_dms(
abs(lat), int(precision))
def add_image_history(self, data):
"""Add arbitrary string to ImageHistory tag."""
self._ef['0th'][piexif.ImageIFD.ImageHistory] = json.dumps(data)
def add_camera_make_model(self, make, model):
''' Add camera make and model.'''
self._ef['0th'][piexif.ImageIFD.Make] = make
self._ef['0th'][piexif.ImageIFD.Model] = model
def add_dop(self, dop, precision=100):
"""Add GPSDOP (float)."""
self._ef["GPS"][piexif.GPSIFD.GPSDOP] = (
int(abs(dop) * precision), precision)
def add_altitude(self, altitude, precision=100):
"""Add altitude (pre is the precision)."""
ref = 0 if altitude > 0 else 1
self._ef["GPS"][piexif.GPSIFD.GPSAltitude] = (
int(abs(altitude) * precision), precision)
self._ef["GPS"][piexif.GPSIFD.GPSAltitudeRef] = ref
def add_direction(self, direction, ref="T", precision=100):
"""Add image direction."""
# normalize direction
direction = direction % 360.0
self._ef["GPS"][piexif.GPSIFD.GPSImgDirection] = (
int(abs(direction) * precision), precision)
self._ef["GPS"][piexif.GPSIFD.GPSImgDirectionRef] = ref
def add_firmware(self,firmware_string):
"""Add firmware version of camera"""
self._ef['0th'][piexif.ImageIFD.Software] = firmware_string
def add_custom_tag(self, value, main_key, tag_key):
try:
self._ef[main_key][tag_key] = value
except:
print("could not set tag {} under {} with value {}".format(
tag_key, main_key, value))
def write(self, filename=None):
"""Save exif data to file."""
if filename is None:
filename = self._filename
exif_bytes = piexif.dump(self._ef)
with open(self._filename, "rb") as fin:
img = fin.read()
try:
piexif.insert(exif_bytes, img, filename)
except IOError:
type, value, traceback = sys.exc_info()
print >> sys.stderr, "Error saving file:", value

245
lib/exifedit.py
View File

@ -1,245 +0,0 @@
import sys
import json
import datetime
import hashlib
import base64
import uuid
from lib.geo import normalize_bearing
from lib.exif import EXIF, verify_exif
from lib.pexif import JpegFile, Rational
import shutil
def create_mapillary_description(filename, username, email, userkey,
upload_hash, sequence_uuid,
interpolated_heading=None,
offset_angle=0.0,
timestamp=None,
orientation=None,
project="",
secret_hash=None,
external_properties=None,
verbose=False):
'''
Check that image file has the required EXIF fields.
Incompatible files will be ignored server side.
'''
# read exif
exif = EXIF(filename)
if not verify_exif(filename):
return False
if orientation is None:
orientation = exif.extract_orientation()
# write the mapillary tag
mapillary_description = {}
# lat, lon of the image, takes precedence over EXIF GPS values
mapillary_description["MAPLongitude"], mapillary_description["MAPLatitude"] = exif.extract_lon_lat()
# altitude of the image, takes precedence over EXIF GPS values, assumed 0 if missing
mapillary_description["MAPAltitude"] = exif.extract_altitude()
# capture time: required date format: 2015_01_14_09_37_01_000, TZ MUST be UTC
if timestamp is None:
timestamp = exif.extract_capture_time()
# The capture time of the image in UTC. Will take precedence over any other time tags in the EXIF
mapillary_description["MAPCaptureTime"] = datetime.datetime.strftime(timestamp, "%Y_%m_%d_%H_%M_%S_%f")[:-3]
# EXIF orientation of the image
mapillary_description["MAPOrientation"] = orientation
heading = exif.extract_direction()
if heading is None:
heading = 0.0
heading = normalize_bearing(interpolated_heading + offset_angle) if interpolated_heading is not None else normalize_bearing(heading + offset_angle)
# bearing of the image
mapillary_description["MAPCompassHeading"] = {"TrueHeading": heading, "MagneticHeading": heading}
# authentication
assert(email is not None or userkey is not None)
if email is not None:
mapillary_description["MAPSettingsEmail"] = email
if username is not None:
mapillary_description["MAPSettingsUsername"] = username
# use this if available, and omit MAPSettingsUsername and MAPSettingsEmail for privacy reasons
if userkey is not None:
mapillary_description["MAPSettingsUserKey"] = userkey
if upload_hash is not None:
settings_upload_hash = hashlib.sha256("%s%s%s" % (upload_hash, email, base64.b64encode(filename))).hexdigest()
# this is not checked in the backend right now, will likely be changed to have user_key instead of email as part
# of the hash
mapillary_description['MAPSettingsUploadHash'] = settings_upload_hash
# a unique photo ID to check for duplicates in the backend in case the image gets uploaded more than once
mapillary_description['MAPPhotoUUID'] = str(uuid.uuid4())
# a sequene ID to make the images go together (order by MAPCaptureTime)
mapillary_description['MAPSequenceUUID'] = str(sequence_uuid)
# The device model
mapillary_description['MAPDeviceModel'] = exif.extract_model()
# The device manufacturer
mapillary_description['MAPDeviceMake'] = exif.extract_make()
if upload_hash is None and secret_hash is not None:
mapillary_description['MAPVideoSecure'] = secret_hash
mapillary_description["MAPSettingsProject"] = project
# external properties (optional)
if external_properties is not None:
# externl proerties can be saved and searched in Mapillary later on
mapillary_description['MAPExternalProperties'] = external_properties
# write to file
if verbose:
print("tag: {0}".format(mapillary_description))
metadata = ExifEdit(filename)
metadata.add_image_description(mapillary_description)
metadata.add_orientation(orientation)
metadata.add_direction(heading)
metadata.write()
def add_mapillary_description(filename, username, email,
project, upload_hash, image_description,
output_file=None):
"""Add Mapillary description tags directly with user info."""
if username is not None:
# write the mapillary tag
image_description["MAPSettingsUploadHash"] = upload_hash
image_description["MAPSettingsEmail"] = email
image_description["MAPSettingsUsername"] = username
settings_upload_hash = hashlib.sha256("%s%s%s" % (upload_hash, email, base64.b64encode(filename))).hexdigest()
image_description['MAPSettingsUploadHash'] = settings_upload_hash
# if this image is part of a projet, the project UUID
image_description["MAPSettingsProject"] = project
assert("MAPSequenceUUID" in image_description)
if output_file is not None:
shutil.copy(filename, output_file)
filename = output_file
# write to file
json_desc = json.dumps(image_description)
metadata = ExifEdit(filename)
metadata.add_image_description(json_desc)
metadata.add_orientation(image_description.get("MAPOrientation", 1))
metadata.add_direction(image_description["MAPCompassHeading"]["TrueHeading"])
metadata.add_lat_lon(image_description["MAPLatitude"], image_description["MAPLongitude"])
date_time = datetime.datetime.strptime(image_description["MAPCaptureTime"]+"000", "%Y_%m_%d_%H_%M_%S_%f")
metadata.add_date_time_original(date_time)
metadata.write()
def add_exif_data(filename, data, output_file=None):
"""Add minimal exif data to an image"""
if output_file is not None:
shutil.copy(filename, output_file)
filename = output_file
metadata = ExifEdit(filename)
metadata.add_orientation(data.get("orientation", 1))
metadata.add_direction(data.get("bearing", 0))
metadata.add_lat_lon(data["lat"], data[