From 5da70b4e995dc8a3de8f5b8bd16e4eb1eb53cf23 Mon Sep 17 00:00:00 2001 From: declerambaul Date: Thu, 3 May 2012 00:01:30 -0400 Subject: [PATCH] Extended scripts for target data. Added dygraph data for mobile pageviews and active editors --- data/datasources/rc/rc_active_editors_target.csv | 83 ++++++++++---------- .../rc/rc_page_requests_mobile_target.csv | 52 ++++++------ data/targets/mobile_target.csv | 48 ++++++------ msc/old_rc_new/new_charts.py | 7 +- msc/old_rc_new/old_to_new.py | 3 +- msc/old_rc_new/targets.py | 80 +++++++++++++++++++ msc/old_rc_new/utils.py | 18 +++- 7 files changed, 189 insertions(+), 102 deletions(-) create mode 100644 msc/old_rc_new/targets.py diff --git a/data/datasources/rc/rc_active_editors_target.csv b/data/datasources/rc/rc_active_editors_target.csv index 5029448..43d3dcb 100644 --- a/data/datasources/rc/rc_active_editors_target.csv +++ b/data/datasources/rc/rc_active_editors_target.csv @@ -1,42 +1,41 @@ -Month,Total,Projected,Target -2009/02/01,86075,, -2009/03/01,90821,, -2009/04/01,86401,, -2009/05/01,89124,, -2009/06/01,87784,, -2009/07/01,87160,, -2009/08/01,87121,, -2009/09/01,84270,, -2009/10/01,86333,, -2009/11/01,85911,, -2009/12/01,84077,, -2010/01/01,90283,, -2010/02/01,85092,, -2010/03/01,89458,, -2010/04/01,86040,, -2010/05/01,87954,, -2010/06/01,82529,, -2010/07/01,80662,, -2010/08/01,82102,, -2010/09/01,78819,, -2010/10/01,80398,, -2010/11/01,79381,, -2010/12/01,78856,, -2011/01/01,88233,88459, -2011/02/01,83462,83788, -2011/03/01,86009,86439,75755 -2011/04/01,81869,84691,79509 -2011/05/01,82985,84864,80853 -2011/06/01,81416,82350,79620 -2011/07/01,81558,81302,79773 -2011/08/01,82313,80482,80139 -2011/09/01,79682,79338,80171 -2011/10/01,80273,80317,82364 -2011/11/01,79607,79145,82365 -2011/12/01,79663,77795,82161 -2012/01/01,85054,81829,90352 -2012/02/01,81792,81227,91016 -2012/03/01,,83135,94536 -2012/04/01,,82208,94867 -2012/05/01,,82376,96471 -2012/06/01,,79935,95000 +Date,Total,Target,Projected +2009/03/01,94462,, +2009/04/01,89657,, +2009/05/01,92258,, +2009/06/01,90811,, +2009/07/01,90087,, +2009/08/01,90230,, +2009/09/01,87247,, +2009/10/01,89479,, +2009/11/01,89136,, +2009/12/01,87244,, +2010/01/01,93499,, +2010/02/01,88339,, +2010/03/01,92644,, +2010/04/01,89153,, +2010/05/01,91110,, +2010/06/01,85521,, +2010/07/01,83521,, +2010/08/01,84987,, +2010/09/01,81750,, +2010/10/01,83464,, +2010/11/01,82729,, +2010/12/01,81820,, +2011/01/01,91309,,88459 +2011/02/01,86580,,83788 +2011/03/01,89064,75755,86439 +2011/04/01,84819,79509,84691 +2011/05/01,85884,80853,84864 +2011/06/01,84349,79620,82350 +2011/07/01,84174,79773,81302 +2011/08/01,85165,80139,80482 +2011/09/01,82627,80171,79338 +2011/10/01,83517,82364,80317 +2011/11/01,82908,82365,79145 +2011/12/01,82757,82161,77795 +2012/01/01,88122,90352,81829 +2012/02/01,84923,91016,81227 +2012/03/01,85092,94536,83135 +2012/04/01,,94867,82208 +2012/05/01,,96471,82376 +2012/06/01,,95000,79935 diff --git a/data/datasources/rc/rc_page_requests_mobile_target.csv b/data/datasources/rc/rc_page_requests_mobile_target.csv index 42900d3..fc03b62 100644 --- a/data/datasources/rc/rc_page_requests_mobile_target.csv +++ b/data/datasources/rc/rc_page_requests_mobile_target.csv @@ -1,26 +1,26 @@ -Month,Total,Target -2010/06/01,207.541927, -2010/07/01,371.237914,372 -2010/08/01,400.198907,401 -2010/09/01,424.568581,425 -2010/10/01,431.946274,433 -2010/11/01,491.301811,492 -2010/12/01,555.316438,556 -2011/01/01,680.095704,670 -2011/02/01,691.989863,690 -2011/03/01,622.231377,620 -2011/04/01,726.090685,726 -2011/05/01,760.513392,785 -2011/06/01,798.945757,848.478495575645 -2011/07/01,854.290939,917.090136884472 -2011/08/01,912.602031,991.250012294266 -2011/09/01,880.256597,1071.40677601373 -2011/10/01,1103.656174,1158.04536237157 -2011/11/01,1315.391306,1251.68991958393 -2011/12/01,1522.864725,1352.90698075894 -2012/01/01,1669.647789,1462.30889132247 -2012/02/01,1651.793186,1580.55751360023 -2012/03/01,,1708.36823096854 -2012/04/01,,1846.51427579798 -2012/05/01,,1995.83140737327 -2012/06/01,,2157.22296809006 +Date,Total,Target +2010/06/01,208280310, +2010/07/01,372132091,372000000 +2010/08/01,400944073,401000000 +2010/09/01,425385736,425000000 +2010/10/01,432613085,433000000 +2010/11/01,491794260,492000000 +2010/12/01,555653967,556000000 +2011/01/01,680527579,670000000 +2011/02/01,692464914,690000000 +2011/03/01,622902258,620000000 +2011/04/01,728233197,726000000 +2011/05/01,762990098,785000000 +2011/06/01,801552556,848478495.575645 +2011/07/01,857407355,917090136.884472 +2011/08/01,916946743,991250012.294266 +2011/09/01,887103215,1071406776.01373 +2011/10/01,1111140204,1158045362.37157 +2011/11/01,1325724262,1251689919.58393 +2011/12/01,1533885014,1352906980.75894 +2012/01/01,1681435126,1462308891.32247 +2012/02/01,1663482355,1580557513.60023 +2012/03/01,1858551326,1708368230.96854 +2012/04/01,,1846514275.79798 +2012/05/01,,1995831407.37327 +2012/06/01,,2157222968.09006 diff --git a/data/targets/mobile_target.csv b/data/targets/mobile_target.csv index 098e947..c940ca8 100644 --- a/data/targets/mobile_target.csv +++ b/data/targets/mobile_target.csv @@ -1,24 +1,24 @@ -Jul-10 385 385 372 372 -Aug-10 414 414 401 401 -Sep-10 425 425 425 425 -Oct-10 447 447 433 433 -Nov-10 492 492 492 492 -Dec-10 574 574 556 556 -Jan-11 693 703 670 681 -Feb-11 644 646 690 692 -Mar-11 641 644 620 623 -Apr-11 726 728 726 728 -May-11 760 788 785 763 -Jun-11 815.661344711728 802 848.478495575645 802 -Jul-11 875.399249022295 886 917.090136884472 857 -Aug-11 939.512274496266 948 991.250012294266 917 -Sep-11 1008.32084893263 887 1071.40677601373 887 -Oct-11 1082.16886781745 1148 1158.04536237157 1111 -Nov-11 1161.42541306467 1326 1251.68991958393 1326 -Dec-11 1246.48659763513 1585 1352.90698075894 1534 -Jan-12 1337.77754525292 1462.30889132247 -Feb-12 1435.75451511335 1580.55751360023 -Mar-12 1540.9071822017 1708.36823096854 -Apr-12 1653.76108461921 1846.51427579798 -May-12 1774.88025014794 1995.83140737327 -Jun-12 1904.87001518152 2157.22296809006 +Jul-10 385 385 372000000 372 +Aug-10 414 414 401000000 401 +Sep-10 425 425 425000000 425 +Oct-10 447 447 433000000 433 +Nov-10 492 492 492000000 492 +Dec-10 574 574 556000000 556 +Jan-11 693 703 670000000 681 +Feb-11 644 646 690000000 692 +Mar-11 641 644 620000000 623 +Apr-11 726 728 726000000 728 +May-11 760 788 785000000 763 +Jun-11 815.661344711728 802 848478495.575645 802 +Jul-11 875.399249022295 886 917090136.884472 857 +Aug-11 939.512274496266 948 991250012.294266 917 +Sep-11 1008.32084893263 887 1071406776.01373 887 +Oct-11 1082.16886781745 1148 1158045362.37157 1111 +Nov-11 1161.42541306467 1326 1251689919.58393 1326 +Dec-11 1246.48659763513 1585 1352906980.75894 1534 +Jan-12 1337.77754525292 1462308891.32247 +Feb-12 1435.75451511335 1580557513.60023 +Mar-12 1540.9071822017 1708368230.96854 +Apr-12 1653.76108461921 1846514275.79798 +May-12 1774.88025014794 1995831407.37327 +Jun-12 1904.87001518152 2157222968.09006 diff --git a/msc/old_rc_new/new_charts.py b/msc/old_rc_new/new_charts.py index bf64cdf..9d196ec 100644 --- a/msc/old_rc_new/new_charts.py +++ b/msc/old_rc_new/new_charts.py @@ -5,7 +5,7 @@ import os from collections import OrderedDict import parse_csv as p - +import targets def mobile_page_requests(): @@ -43,7 +43,8 @@ def mobile_page_requests(): - def create_all(): - mobile_page_requests() \ No newline at end of file + mobile_page_requests() + + targets.create_all() \ No newline at end of file diff --git a/msc/old_rc_new/old_to_new.py b/msc/old_rc_new/old_to_new.py index f5fe94a..cee76b9 100644 --- a/msc/old_rc_new/old_to_new.py +++ b/msc/old_rc_new/old_to_new.py @@ -18,10 +18,9 @@ import new_charts import comscore - def create_all(): old_charts.create_all() - # new_charts.create_all() + new_charts.create_all() comscore.create_all() if __name__ == '__main__': diff --git a/msc/old_rc_new/targets.py b/msc/old_rc_new/targets.py new file mode 100644 index 0000000..3cad0b0 --- /dev/null +++ b/msc/old_rc_new/targets.py @@ -0,0 +1,80 @@ +''' +Creates dygrap csv files for target/benchmarks +''' +import os +from collections import OrderedDict,defaultdict + +import old_to_new +import utils + +def extractData(fn, sep,format,fields): + ''' + :arg sep: seperater + :arg format: format of date (which is on the first position) + :arg fields: list of tuples ('name',index), e.g. ('target',2) + ''' + data = defaultdict(lambda : defaultdict(str)) + # data = defaultdict(dict) + for line in open(fn,'r'): + # first pos is always data + if line[-1]!='\n': + line+='\n' + + vals = line[:-1].split(sep) + ts = utils.to_ts(vals[0],format) + + for n,i in fields: + data[n][ts] = str(vals[i]) + + return data + +def dygraphOutput(out_fn,tss,actual,target,a_names,t_names): + + with open(os.path.join(old_to_new.output_folder,out_fn),'w') as out: + headers = ["Date"]+a_names+t_names #[f[0] for f in a_fields]+[f[0] for f in t_fields] + out.write(','.join(headers)+'\n') + + for ts in tss: + #ordered list of timestamps + # print actual['Total'] + vals = [utils.dygraph_date(ts)]+[actual[n][ts] for n in a_names]+[target[n][ts] for n in t_names] + out.write(','.join(vals)+'\n') + +def active_editors_target(): + # input files + metricfile = '../../data/datasources/rc/rc_active_editors_count.csv' + targetfile = '../../data/targets/editor_target.csv' + + # time range + # TODO, extract automatically + min_ts = '200903' + max_ts = '201206' + tss,tss_i = utils.create_time_stamps_month(fromym=min_ts,toym=max_ts) + + + target = extractData(targetfile, '\t','MMM-YY',[('Projected',1),('Target',2)]) + actual = extractData(metricfile, ',','YYYY/MM/DD',[('Total',1)]) + + dygraphOutput('rc_active_editors_target.csv',tss,actual,target,['Total'],['Target','Projected']) + +def mobile_pageviews_target(): + # input files + metricfile = '../../data/datasources/rc/rc_page_requests_mobile.csv' + targetfile = '../../data/targets/mobile_target.csv' + + # time range + # TODO, extract automatically + min_ts = '201006' + max_ts = '201206' + tss,tss_i = utils.create_time_stamps_month(fromym=min_ts,toym=max_ts) + + + target = extractData(targetfile, '\t','MMM-YY',[('Target',3)]) + actual = extractData(metricfile, ',','YYYY/MM/DD',[('Total',1)]) + + dygraphOutput('rc_page_requests_mobile_target.csv',tss,actual,target,['Total'],['Target']) + +def create_all(): + active_editors_target() + mobile_pageviews_target() + diff --git a/msc/old_rc_new/utils.py b/msc/old_rc_new/utils.py index c23173d..b4a13aa 100644 --- a/msc/old_rc_new/utils.py +++ b/msc/old_rc_new/utils.py @@ -8,20 +8,28 @@ import logging months_dict = {'Mar': '03', 'Feb': '02', 'Aug': '08', 'Sep': '09', 'Apr': '04', 'Jun': '06', 'Jul': '07', 'Jan': '01', 'May': '05', 'Nov': '11', 'Dec': '12', 'Oct': '10'} - -def fix_dates(dates,format): +def fix_date(t,format): '''Formats the date from Jan/2001 into 2001/01/01 style''' if format == 'MMM-YY': - return ['20%s/%s/01'%(t[4:], months_dict[t[:3]]) for t in dates] + return '20%s/%s/01'%(t[4:], months_dict[t[:3]]) elif format == 'MM/YYYY': - return ['%s/%s/01'%(t[3:],t[:2]) for t in dates] + return '%s/%s/01'%(t[3:],t[:2]) elif format == 'YYYY-MM': - return ['%s/%s/01'%(t[:4],t[5:]) for t in dates] + return '%s/%s/01'%(t[:4],t[5:]) +def fix_dates(dates,format): + return [fix_date(t,format) for t in dates] + def dygraph_date(ts): return '%s/%s/01'%(ts[:4],ts[4:]) +def to_ts(date,format): + '''Formats the date from Jan/2001 or simple ts style''' + if format != 'YYYY/MM/DD': + date = fix_date(date,format) + return '%s%s'%(date[:4],date[5:7]) + def create_time_stamps_month(fromym='200101',toym='201012'): ''' -- 1.7.0.4