Skip to content

Commit

Permalink
Finished provenance; slight detail changes
Browse files Browse the repository at this point in the history
  • Loading branch information
cjoeml committed Oct 4, 2017
1 parent d161b74 commit 53e935f
Show file tree
Hide file tree
Showing 9 changed files with 125 additions and 807 deletions.
33 changes: 14 additions & 19 deletions bkin18_cjoe/emergency_traffic_aggregate.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,36 +79,31 @@ def provenance(doc = prov.model.ProvDocument(), startTime = None, endTime = None
doc.add_namespace('ont', 'http://datamechanics.io/ontology#') # 'Extension', 'DataResource', 'DataSet', 'Retrieval', 'Query', or 'Computation'.
doc.add_namespace('log', 'http://datamechanics.io/log/') # The event log.
doc.add_namespace('bdp', 'http://bostonopendata-boston.opendata.arcgis.com/datasets/')
doc.add_namespace('hdv', 'https://dataverse.harvard.edu/dataset.xhtml')

this_script = doc.agent('alg:bkin18_cjoe#emergency_traffic_aggregate',
{ prov.model.PROV_TYPE:prov.model.PROV['SoftwareAgent'], 'ont:Extension':'py'})

this_script = doc.agent('alg:bkin18_cjoe#emergency_traffic_aggregate', {prov.model.PROV_TYPE:prov.model.PROV['SoftwareAgent'], 'ont:Extension':'py'})
this_run = doc.activity('log:a'+str(uuid.uuid4()), startTime, endTime,
{ prov.model.PROV_TYPE:'ont:Retrieval', 'ont:Query':'.find()'})

#resource = doc.entity('bdp:4f3e4492e36f4907bcd307b131afe4a5_0',
# {'prov:label':'311, Service Requests',
# prov.model.PROV_TYPE:'ont:DataResource', 'bdp:Extension':'geojson'})
selection_input = doc.entity('dat:bkin18_cjoe.emergency_traffic_selection',
{ prov.model.PROV_LABEL:'Emergency Traffic Selection', prov.model.PROV_TYPE:'ont:DataSet'})

## Work on this later
this_run = doc.activity('log:a'+str(uuid.uuid4()), startTime, endTime, { prov.model.PROV_TYPE:'ont:Retrieval'})#, 'ont:Query':'?type=Animal+Found&$select=type,latitude,longitude,OPEN_DT'})
output = doc.entity('dat:bkin18_cjoe.emergency_traffic_aggregate',
{ prov.model.PROV_LABEL:'Emergency Traffic Aggregation', prov.model.PROV_TYPE:'ont:DataSet'})

route_activity = doc.activity('log:uuid'+str(uuid.uuid4()), startTime, endTime)

routes = doc.entity('dat:bkin18_cjoe#emergency_traffic_aggregate', {prov.model.PROV_LABEL:'Emergency Traffic Selection', prov.model.PROV_TYPE:'ont:DataSet'})

doc.wasAssociatedWith(route_activity, this_script)
#doc.usage(route_activity, resource, startTime, None, {prov.model.PROV_TYPE:'ont:Retrieval'})
#doc.wasAttributedTo(found, this_script)

doc.wasAttributedTo(routes, this_script)
doc.wasGeneratedBy(routes, route_activity, endTime)
#doc.wasDerivedFrom(routes, resource, this_run, this_run, this_run)
doc.wasAssociatedWith(this_run , this_script)
doc.used(this_run, selection_input, startTime)

doc.wasAttributedTo(output, this_script)
doc.wasGeneratedBy(output, this_run, endTime)
doc.wasDerivedFrom(output, selection_input, this_run, this_run, this_run)

repo.logout()


return doc


emergency_traffic_aggregate.execute()
doc = emergency_traffic_aggregate.provenance()
print(doc.get_provn())
Expand Down
37 changes: 21 additions & 16 deletions bkin18_cjoe/emergency_traffic_selection.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ class emergency_traffic_selection(dml.Algorithm):
reads = ['bkin18_cjoe.emergency_routes', 'bkin18_cjoe.traffic_signals']
writes = ['bkin18_cjoe.emergency_traffic_selection']


@staticmethod
def execute(trial=False):
startTime = datetime.datetime.now()
Expand Down Expand Up @@ -67,31 +66,37 @@ def provenance(doc = prov.model.ProvDocument(), startTime = None, endTime = None
doc.add_namespace('ont', 'http://datamechanics.io/ontology#') # 'Extension', 'DataResource', 'DataSet', 'Retrieval', 'Query', or 'Computation'.
doc.add_namespace('log', 'http://datamechanics.io/log/') # The event log.
doc.add_namespace('bdp', 'http://bostonopendata-boston.opendata.arcgis.com/datasets/')
doc.add_namespace('hdv', 'https://dataverse.harvard.edu/dataset.xhtml')

## Agent
this_script = doc.agent('alg:bkin18_cjoe#emergency_traffic_selection',
{prov.model.PROV_TYPE:prov.model.PROV['SoftwareAgent'], 'ont:Extension':'py'})

this_script = doc.agent('alg:bkin18_cjoe#emergency_traffic_selection', {prov.model.PROV_TYPE:prov.model.PROV['SoftwareAgent'], 'ont:Extension':'py'})
## Activity
this_run = doc.activity('log:a'+str(uuid.uuid4()), startTime, endTime,
{ prov.model.PROV_TYPE:'ont:Retrieval', 'ont:Query':'.find()'})

#resource = doc.entity('bdp:4f3e4492e36f4907bcd307b131afe4a5_0',
# {'prov:label':'311, Service Requests',
# prov.model.PROV_TYPE:'ont:DataResource', 'bdp:Extension':'geojson'})
## Entities
route_input = doc.entity('dat:bkin18_cjoe.emergency_routes',
{ prov.model.PROV_LABEL:'Emergency Traffic Selection', prov.model.PROV_TYPE:'ont:DataSet'})

## Work on this later
this_run = doc.activity('log:a'+str(uuid.uuid4()), startTime, endTime, { prov.model.PROV_TYPE:'ont:Retrieval'})#, 'ont:Query':'?type=Animal+Found&$select=type,latitude,longitude,OPEN_DT'})
traffic_input = doc.entity('dat:bkin18_cjoe.traffic_signals',
{ prov.model.PROV_LABEL:'Emergency Traffic Selection', prov.model.PROV_TYPE:'ont:DataSet'})

# route_activity = doc.activity('log:uuid'+str(uuid.uuid4()), startTime, endTime)
output = doc.entity('dat:bkin18_cjoe.emergency_traffic_selection',
{ prov.model.PROV_LABEL:'Emergency Traffic Selection', prov.model.PROV_TYPE:'ont:DataSet'})

routes = doc.entity('dat:bkin18_cjoe#emergency_traffic_selection', {prov.model.PROV_LABEL:'Emergency Traffic Selection', prov.model.PROV_TYPE:'ont:DataSet'})
doc.wasAssociatedWith(this_run , this_script)
doc.used(this_run, route_input, startTime)
doc.used(this_run, traffic_input, startTime)

doc.wasAssociatedWith(routes, this_script)
#doc.usage(route_activity, resource, startTime, None, {prov.model.PROV_TYPE:'ont:Retrieval'})

doc.wasAttributedTo(routes, this_script)
doc.wasGeneratedBy(routes, this_run, endTime)
#doc.wasDerivedFrom(routes, resource, this_run, this_run, this_run)
doc.wasAttributedTo(output, this_script)
doc.wasGeneratedBy(output, this_run, endTime)
doc.wasDerivedFrom(output, route_input, this_run, this_run, this_run)
doc.wasDerivedFrom(output, traffic_input, this_run, this_run, this_run)

repo.logout()


return doc


Expand Down
30 changes: 16 additions & 14 deletions bkin18_cjoe/get_census.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,24 +73,26 @@ def provenance(doc = prov.model.ProvDocument(), startTime = None, endTime = None
doc.add_namespace('ont', 'http://datamechanics.io/ontology#') # 'Extension', 'DataResource', 'DataSet', 'Retrieval', 'Query', or 'Computation'.
doc.add_namespace('log', 'http://datamechanics.io/log/') # The event log.
doc.add_namespace('bdp', 'http://bostonopendata-boston.opendata.arcgis.com/datasets/')
doc.add_namespace('hdv', 'https://dataverse.harvard.edu/dataset.xhtml')

this_script = doc.agent('alg:bkin18_cjoe#get_census',
{ prov.model.PROV_TYPE:prov.model.PROV['SoftwareAgent'], 'ont:Extension':'py'})

this_script = doc.agent('alg:bkin18_cjoe#get_census', {prov.model.PROV_TYPE:prov.model.PROV['SoftwareAgent'], 'ont:Extension':'py'})

## I don't actually know what the key is to this (bdp:wc8w-nujj)
resource = doc.entity('bdp:DVN_FI1YED', { 'prov:label':'311, Service Requests', prov.model.PROV_TYPE:'ont:DataResource', 'ont:Extension':'csv'})

## Work on this later
this_run = doc.activity('log:a'+str(uuid.uuid4()), startTime, endTime, { prov.model.PROV_TYPE:'ont:Retrieval'})#, 'ont:Query':'?type=Animal+Found&$select=type,latitude,longitude,OPEN_DT'})
this_run = doc.activity('log:a'+str(uuid.uuid4()), startTime, endTime,
{ prov.model.PROV_TYPE:'ont:Retrieval'})#, 'ont:Query':'?type=Animal+Found&$select=type,latitude,longitude,OPEN_DT'})

routes = doc.entity('dat:bkin18_cjoe#census', {prov.model.PROV_LABEL:'Census', prov.model.PROV_TYPE:'ont:DataSet'})
census_input = doc.entity('hdv:DVN_FI1YED',
{ 'prov:label':'311, Service Requests', prov.model.PROV_TYPE:'ont:DataResource', 'ont:Extension':'csv', 'ont:Query':'??persistentId=doi:10.7910'})

output = doc.entity('dat:bkin18_cjoe#census',
{ prov.model.PROV_LABEL:'Census', prov.model.PROV_TYPE:'ont:DataSet'})

doc.wasAssociatedWith(routes, this_script)
doc.usage(routes, resource, startTime, None, {prov.model.PROV_TYPE:'ont:Retrieval'})
doc.wasAttributedTo(routes, this_script)
doc.wasGeneratedBy(routes, this_run, endTime)
doc.wasDerivedFrom(routes, resource, this_run, this_run, this_run)
doc.wasAssociatedWith(this_run , this_script)
doc.used(this_run, census_input, startTime)

doc.wasAttributedTo(output, this_script)
doc.wasGeneratedBy(output, this_run, endTime)
doc.wasDerivedFrom(output, census_input, this_run, this_run, this_run)

repo.logout()

Expand Down
25 changes: 13 additions & 12 deletions bkin18_cjoe/get_roads.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,27 +73,28 @@ def provenance(doc = prov.model.ProvDocument(), startTime = None, endTime = None
doc.add_namespace('ont', 'http://datamechanics.io/ontology#') # 'Extension', 'DataResource', 'DataSet', 'Retrieval', 'Query', or 'Computation'.
doc.add_namespace('log', 'http://datamechanics.io/log/') # The event log.
doc.add_namespace('bdp', 'http://bostonopendata-boston.opendata.arcgis.com/datasets/')
doc.add_namespace('hdv', 'https://dataverse.harvard.edu/dataset.xhtml')

this_script = doc.agent('alg:bkin18_cjoe#get_roads',
{ prov.model.PROV_TYPE:prov.model.PROV['SoftwareAgent'], 'ont:Extension':'py'})

this_script = doc.agent('alg:bkin18_cjoe#get_roads', {prov.model.PROV_TYPE:prov.model.PROV['SoftwareAgent'], 'ont:Extension':'py'})
this_run = doc.activity('log:a'+str(uuid.uuid4()), startTime, endTime,
{ prov.model.PROV_TYPE:'ont:Retrieval'})#, 'ont:Query':'?type=Animal+Found&$select=type,latitude,longitude,OPEN_DT'})

resource = doc.entity('bdp:DVN_OV5PXF', { 'prov:label':'311, Service Requests', prov.model.PROV_TYPE:'ont:DataResource', 'ont:Extension':'geojson'})
road_input = doc.entity('hdv:DVN_OV5PXF',
{ 'prov:label':'311, Service Requests', prov.model.PROV_TYPE:'ont:DataResource', 'ont:Extension':'csv', 'ont:Query':'?persistentId=doi:10.7910'})

this_run = doc.activity('log:a'+str(uuid.uuid4()), startTime, endTime, { prov.model.PROV_TYPE:'ont:Retrieval'})#, 'ont:Query':'?type=Animal+Found&$select=type,latitude,longitude,OPEN_DT'})

routes = doc.entity('dat:bkin18_cjoe#roads', {prov.model.PROV_LABEL:'Roads', prov.model.PROV_TYPE:'ont:DataSet'})
output = doc.entity('dat:bkin18_cjoe.roads', {prov.model.PROV_LABEL:'Roads', prov.model.PROV_TYPE:'ont:DataSet'})

doc.wasAssociatedWith(routes, this_script)
doc.usage(routes, resource, startTime, None, {prov.model.PROV_TYPE:'ont:Retrieval'})
# doc.wasAttributedTo(found, this_script)
doc.wasAssociatedWith(this_run , this_script)
doc.used(this_run, road_input, startTime)

doc.wasAttributedTo(routes, this_script)
doc.wasGeneratedBy(routes, this_run, endTime)
doc.wasDerivedFrom(routes, resource, this_run, this_run, this_run)
doc.wasAttributedTo(output, this_script)
doc.wasGeneratedBy(output, this_run, endTime)
doc.wasDerivedFrom(output, road_input, this_run, this_run, this_run)

repo.logout()


return doc


Expand Down
28 changes: 14 additions & 14 deletions bkin18_cjoe/neighborhood_code.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,27 +62,27 @@ def provenance(doc = prov.model.ProvDocument(), startTime = None, endTime = None
doc.add_namespace('ont', 'http://datamechanics.io/ontology#') # 'Extension', 'DataResource', 'DataSet', 'Retrieval', 'Query', or 'Computation'.
doc.add_namespace('log', 'http://datamechanics.io/log/') # The event log.
doc.add_namespace('bdp', 'http://bostonopendata-boston.opendata.arcgis.com/datasets/')
doc.add_namespace('hdv', 'https://dataverse.harvard.edu/dataset.xhtml')


this_script = doc.agent('alg:bkin18_cjoe#neighborhoods', {prov.model.PROV_TYPE:prov.model.PROV['SoftwareAgent'], 'ont:Extension':'py'})
this_script = doc.agent('alg:bkin18_cjoe#neighborhood_code',
{ prov.model.PROV_TYPE:prov.model.PROV['SoftwareAgent'], 'ont:Extension':'py'})

resource = doc.entity('bdp:3525b0ee6e6b427f9aab5d0a1d0a1a28_0.',
this_run = doc.activity('log:a'+str(uuid.uuid4()), startTime, endTime,
{ prov.model.PROV_TYPE:'ont:Retrieval'})

neighborhood_input = doc.entity('bdp:3525b0ee6e6b427f9aab5d0a1d0a1a28_0.',
{'prov:label':'311, Service Requests',
prov.model.PROV_TYPE:'ont:DataResource', 'bdp:Extension':'geojson'})
## Work on this later
this_run = doc.activity('log:a'+str(uuid.uuid4()), startTime, endTime, { prov.model.PROV_TYPE:'ont:Retrieval'})#, 'ont:Query':'?type=Animal+Found&$select=type,latitude,longitude,OPEN_DT'})

neighborhood_activity = doc.activity('log:uuid'+str(uuid.uuid4()), startTime, endTime)

neighborhoods = doc.entity('dat:bkin18_cjoe#neighborhoods', {prov.model.PROV_LABEL:'Neighborhoods', prov.model.PROV_TYPE:'ont:DataSet'})
output = doc.entity('dat:bkin18_cjoe.neighborhoods',
{ prov.model.PROV_LABEL:'Neighborhoods', prov.model.PROV_TYPE:'ont:DataSet'})

doc.wasAssociatedWith(neighborhood_activity, this_script)
doc.usage(neighborhood_activity, resource, startTime, None, {prov.model.PROV_TYPE:'ont:Retrieval'})
# doc.wasAttributedTo(found, this_script)
doc.wasAssociatedWith(this_run , this_script)
doc.used(this_run, neighborhood_input, startTime)

doc.wasAttributedTo(neighborhoods, this_script)
doc.wasGeneratedBy(neighborhoods, neighborhood_activity, endTime)
doc.wasDerivedFrom(neighborhoods, resource, this_run, this_run, this_run)
doc.wasAttributedTo(output, this_script)
doc.wasGeneratedBy(output, this_run, endTime)
doc.wasDerivedFrom(output, neighborhood_input, this_run, this_run, this_run)

repo.logout()

Expand Down
31 changes: 14 additions & 17 deletions bkin18_cjoe/roads_aggregate.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,29 +67,26 @@ def provenance(doc = prov.model.ProvDocument(), startTime = None, endTime = None
doc.add_namespace('ont', 'http://datamechanics.io/ontology#') # 'Extension', 'DataResource', 'DataSet', 'Retrieval', 'Query', or 'Computation'.
doc.add_namespace('log', 'http://datamechanics.io/log/') # The event log.
doc.add_namespace('bdp', 'http://bostonopendata-boston.opendata.arcgis.com/datasets/')
doc.add_namespace('hdv', 'https://dataverse.harvard.edu/dataset.xhtml')

this_script = doc.agent('alg:bkin18_cjoe#roads_aggregate',
{ prov.model.PROV_TYPE:prov.model.PROV['SoftwareAgent'], 'ont:Extension':'py'})

this_script = doc.agent('alg:bkin18_cjoe#roads_aggregate', {prov.model.PROV_TYPE:prov.model.PROV['SoftwareAgent'], 'ont:Extension':'py'})
this_run = doc.activity('log:a'+str(uuid.uuid4()), startTime, endTime,
{ prov.model.PROV_TYPE:'ont:Retrieval'})#, 'ont:Query':'?type=Animal+Found&$select=type,latitude,longitude,OPEN_DT'})

#resource = doc.entity('bdp:4f3e4492e36f4907bcd307b131afe4a5_0',
# {'prov:label':'311, Service Requests',
# prov.model.PROV_TYPE:'ont:DataResource', 'bdp:Extension':'geojson'})
road_input = doc.entity('dat:bkin18_cjoe.roads',
{ prov.model.PROV_LABEL:'Roads', prov.model.PROV_TYPE:'ont:DataSet', 'ont:Query':'.find()'})

## Work on this later
this_run = doc.activity('log:a'+str(uuid.uuid4()), startTime, endTime, { prov.model.PROV_TYPE:'ont:Retrieval'})#, 'ont:Query':'?type=Animal+Found&$select=type,latitude,longitude,OPEN_DT'})
output = doc.entity('dat:bkin18_cjoe.roads_aggregate',
{ prov.model.PROV_LABEL:'Emergency Traffic Selection', prov.model.PROV_TYPE:'ont:DataSet'})

route_activity = doc.activity('log:uuid'+str(uuid.uuid4()), startTime, endTime)

routes = doc.entity('dat:bkin18_cjoe#roads_aggregate', {prov.model.PROV_LABEL:'Emergency Traffic Selection', prov.model.PROV_TYPE:'ont:DataSet'})

doc.wasAssociatedWith(route_activity, this_script)
#doc.usage(route_activity, resource, startTime, None, {prov.model.PROV_TYPE:'ont:Retrieval'})
#doc.wasAttributedTo(found, this_script)

doc.wasAttributedTo(routes, this_script)
doc.wasGeneratedBy(routes, route_activity, endTime)
#doc.wasDerivedFrom(routes, resource, this_run, this_run, this_run)
doc.wasAssociatedWith(this_run, this_script)
doc.used(this_run, road_input, startTime)

doc.wasAttributedTo(output, this_script)
doc.wasGeneratedBy(output, this_run, endTime)
doc.wasDerivedFrom(output, road_input, this_run, this_run, this_run)

repo.logout()

Expand Down
32 changes: 15 additions & 17 deletions bkin18_cjoe/snow_emergency_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,35 +60,33 @@ def provenance(doc = prov.model.ProvDocument(), startTime = None, endTime = None
doc.add_namespace('ont', 'http://datamechanics.io/ontology#') # 'Extension', 'DataResource', 'DataSet', 'Retrieval', 'Query', or 'Computation'.
doc.add_namespace('log', 'http://datamechanics.io/log/') # The event log.
doc.add_namespace('bdp', 'http://bostonopendata-boston.opendata.arcgis.com/datasets/')
doc.add_namespace('hdv', 'https://dataverse.harvard.edu/dataset.xhtml')


this_script = doc.agent('alg:bkin18_cjoe#snow_emergency_routes', {prov.model.PROV_TYPE:prov.model.PROV['SoftwareAgent'], 'ont:Extension':'py'})
this_script = doc.agent('alg:bkin18_cjoe#snow_emergency_routes',
{ prov.model.PROV_TYPE:prov.model.PROV['SoftwareAgent'], 'ont:Extension':'py'})

## I don't actually know what the key is to this (bdp:wc8w-nujj)
# resource = doc.entity('bdp:wc8w-nujj', { 'prov:label':'311, Service Requests', prov.model.PROV_TYPE:'ont:DataResource', 'ont:Extension':'geojson'})
resource = doc.entity('bdp:4f3e4492e36f4907bcd307b131afe4a5_0',
this_run = doc.activity('log:a'+str(uuid.uuid4()), startTime, endTime,
{ prov.model.PROV_TYPE:'ont:Retrieval'})#, 'ont:Query':'?type=Animal+Found&$select=type,latitude,longitude,OPEN_DT'})

route_input = doc.entity('bdp:4f3e4492e36f4907bcd307b131afe4a5_0',
{'prov:label':'311, Service Requests',
prov.model.PROV_TYPE:'ont:DataResource', 'bdp:Extension':'geojson'})

## Work on this later
this_run = doc.activity('log:a'+str(uuid.uuid4()), startTime, endTime, { prov.model.PROV_TYPE:'ont:Retrieval'})#, 'ont:Query':'?type=Animal+Found&$select=type,latitude,longitude,OPEN_DT'})

route_activity = doc.activity('log:uuid'+str(uuid.uuid4()), startTime, endTime)

routes = doc.entity('dat:bkin18_cjoe#emergency_routes', {prov.model.PROV_LABEL:'Snow Emergency Routes', prov.model.PROV_TYPE:'ont:DataSet'})
output = doc.entity('dat:bkin18_cjoe.emergency_routes',
{ prov.model.PROV_LABEL:'Snow Emergency Routes', prov.model.PROV_TYPE:'ont:DataSet'})

doc.wasAssociatedWith(route_activity, this_script)
doc.usage(route_activity, resource, startTime, None, {prov.model.PROV_TYPE:'ont:Retrieval'})
# doc.wasAttributedTo(found, this_script)
doc.wasAssociatedWith(this_run, this_script)

doc.wasAttributedTo(routes, this_script)
doc.wasGeneratedBy(routes, route_activity, endTime)
doc.wasDerivedFrom(routes, resource, this_run, this_run, this_run)
doc.used(this_run, route_input, startTime)
# doc.usage(routes, resource, startTime, None, {prov.model.PROV_TYPE:'ont:Retrieval'})

doc.wasAttributedTo(output, this_script)
doc.wasGeneratedBy(output, this_run, endTime)
doc.wasDerivedFrom(output, route_input, this_run, this_run, this_run)

repo.logout()


return doc


Expand Down
Loading

0 comments on commit 53e935f

Please sign in to comment.