From 59aeb026b5a5d1502e60956d0c5b10f81e6ef03d Mon Sep 17 00:00:00 2001 From: declerambaul Date: Mon, 14 May 2012 13:52:06 -0400 Subject: [PATCH] In attempt to style lines differently, I added fixed error bars to target lines, specified as a percentage of the max value encountered. Does not look great to say the least. --- data/datasources/rc/rc_active_editors_target.csv | 80 +++--- .../rc/rc_page_requests_mobile_target.csv | 50 ++-- data/graphs/active_editors_target.json | 2 +- data/graphs/pageviews_mobile_target.json | 2 +- msc/old_rc_new/new_charts.py | 3 - msc/old_rc_new/old_to_new.py | 12 +- msc/old_rc_new/targets.py | 72 ++++-- msc/old_rc_new/utils.py | 288 ++++++++++---------- 8 files changed, 274 insertions(+), 235 deletions(-) diff --git a/data/datasources/rc/rc_active_editors_target.csv b/data/datasources/rc/rc_active_editors_target.csv index 43d3dcb..6e482f3 100644 --- a/data/datasources/rc/rc_active_editors_target.csv +++ b/data/datasources/rc/rc_active_editors_target.csv @@ -1,41 +1,41 @@ Date,Total,Target,Projected -2009/03/01,94462,, -2009/04/01,89657,, -2009/05/01,92258,, -2009/06/01,90811,, -2009/07/01,90087,, -2009/08/01,90230,, -2009/09/01,87247,, -2009/10/01,89479,, -2009/11/01,89136,, -2009/12/01,87244,, -2010/01/01,93499,, -2010/02/01,88339,, -2010/03/01,92644,, -2010/04/01,89153,, -2010/05/01,91110,, -2010/06/01,85521,, -2010/07/01,83521,, -2010/08/01,84987,, -2010/09/01,81750,, -2010/10/01,83464,, -2010/11/01,82729,, -2010/12/01,81820,, -2011/01/01,91309,,88459 -2011/02/01,86580,,83788 -2011/03/01,89064,75755,86439 -2011/04/01,84819,79509,84691 -2011/05/01,85884,80853,84864 -2011/06/01,84349,79620,82350 -2011/07/01,84174,79773,81302 -2011/08/01,85165,80139,80482 -2011/09/01,82627,80171,79338 -2011/10/01,83517,82364,80317 -2011/11/01,82908,82365,79145 -2011/12/01,82757,82161,77795 -2012/01/01,88122,90352,81829 -2012/02/01,84923,91016,81227 -2012/03/01,85092,94536,83135 -2012/04/01,,94867,82208 -2012/05/01,,96471,82376 -2012/06/01,,95000,79935 +2009/03/01,94462;94462;94462,, +2009/04/01,89657;89657;89657,, +2009/05/01,92258;92258;92258,, +2009/06/01,90811;90811;90811,, +2009/07/01,90087;90087;90087,, +2009/08/01,90230;90230;90230,, +2009/09/01,87247;87247;87247,, +2009/10/01,89479;89479;89479,, +2009/11/01,89136;89136;89136,, +2009/12/01,87244;87244;87244,, +2010/01/01,93499;93499;93499,, +2010/02/01,88339;88339;88339,, +2010/03/01,92644;92644;92644,, +2010/04/01,89153;89153;89153,, +2010/05/01,91110;91110;91110,, +2010/06/01,85521;85521;85521,, +2010/07/01,83521;83521;83521,, +2010/08/01,84987;84987;84987,, +2010/09/01,81750;81750;81750,, +2010/10/01,83464;83464;83464,, +2010/11/01,82729;82729;82729,, +2010/12/01,81820;81820;81820,, +2011/01/01,91309;91309;91309,,88459;88459;88459 +2011/02/01,86580;86580;86580,,83788;83788;83788 +2011/03/01,89064;89064;89064,73825.58;75755;77684.42,86439;86439;86439 +2011/04/01,84819;84819;84819,77579.58;79509;81438.42,84691;84691;84691 +2011/05/01,85884;85884;85884,78923.58;80853;82782.42,84864;84864;84864 +2011/06/01,84349;84349;84349,77690.58;79620;81549.42,82350;82350;82350 +2011/07/01,84174;84174;84174,77843.58;79773;81702.42,81302;81302;81302 +2011/08/01,85165;85165;85165,78209.58;80139;82068.42,80482;80482;80482 +2011/09/01,82627;82627;82627,78241.58;80171;82100.42,79338;79338;79338 +2011/10/01,83517;83517;83517,80434.58;82364;84293.42,80317;80317;80317 +2011/11/01,82908;82908;82908,80435.58;82365;84294.42,79145;79145;79145 +2011/12/01,82757;82757;82757,80231.58;82161;84090.42,77795;77795;77795 +2012/01/01,88122;88122;88122,88422.58;90352;92281.42,81829;81829;81829 +2012/02/01,84923;84923;84923,89086.58;91016;92945.42,81227;81227;81227 +2012/03/01,85092;85092;85092,92606.58;94536;96465.42,83135;83135;83135 +2012/04/01,,92937.58;94867;96796.42,82208;82208;82208 +2012/05/01,,94541.58;96471;98400.42,82376;82376;82376 +2012/06/01,,93070.58;95000;96929.42,79935;79935;79935 diff --git a/data/datasources/rc/rc_page_requests_mobile_target.csv b/data/datasources/rc/rc_page_requests_mobile_target.csv index fc03b62..b951cb4 100644 --- a/data/datasources/rc/rc_page_requests_mobile_target.csv +++ b/data/datasources/rc/rc_page_requests_mobile_target.csv @@ -1,26 +1,26 @@ Date,Total,Target -2010/06/01,208280310, -2010/07/01,372132091,372000000 -2010/08/01,400944073,401000000 -2010/09/01,425385736,425000000 -2010/10/01,432613085,433000000 -2010/11/01,491794260,492000000 -2010/12/01,555653967,556000000 -2011/01/01,680527579,670000000 -2011/02/01,692464914,690000000 -2011/03/01,622902258,620000000 -2011/04/01,728233197,726000000 -2011/05/01,762990098,785000000 -2011/06/01,801552556,848478495.575645 -2011/07/01,857407355,917090136.884472 -2011/08/01,916946743,991250012.294266 -2011/09/01,887103215,1071406776.01373 -2011/10/01,1111140204,1158045362.37157 -2011/11/01,1325724262,1251689919.58393 -2011/12/01,1533885014,1352906980.75894 -2012/01/01,1681435126,1462308891.32247 -2012/02/01,1663482355,1580557513.60023 -2012/03/01,1858551326,1708368230.96854 -2012/04/01,,1846514275.79798 -2012/05/01,,1995831407.37327 -2012/06/01,,2157222968.09006 +2010/06/01,208280310;208280310;208280310, +2010/07/01,372132091;372132091;372132091,285711081.276;372000000;458288918.724 +2010/08/01,400944073;400944073;400944073,314711081.276;401000000;487288918.724 +2010/09/01,425385736;425385736;425385736,338711081.276;425000000;511288918.724 +2010/10/01,432613085;432613085;432613085,346711081.276;433000000;519288918.724 +2010/11/01,491794260;491794260;491794260,405711081.276;492000000;578288918.724 +2010/12/01,555653967;555653967;555653967,469711081.276;556000000;642288918.724 +2011/01/01,680527579;680527579;680527579,583711081.276;670000000;756288918.724 +2011/02/01,692464914;692464914;692464914,603711081.276;690000000;776288918.724 +2011/03/01,622902258;622902258;622902258,533711081.276;620000000;706288918.724 +2011/04/01,728233197;728233197;728233197,639711081.276;726000000;812288918.724 +2011/05/01,762990098;762990098;762990098,698711081.276;785000000;871288918.724 +2011/06/01,801552556;801552556;801552556,762189576.852;848478495.575645;934767414.299 +2011/07/01,857407355;857407355;857407355,830801218.161;917090136.884472;1003379055.61 +2011/08/01,916946743;916946743;916946743,904961093.571;991250012.294266;1077538931.02 +2011/09/01,887103215;887103215;887103215,985117857.29;1071406776.01373;1157695694.74 +2011/10/01,1111140204;1111140204;1111140204,1071756443.65;1158045362.37157;1244334281.1 +2011/11/01,1325724262;1325724262;1325724262,1165401000.86;1251689919.58393;1337978838.31 +2011/12/01,1533885014;1533885014;1533885014,1266618062.04;1352906980.75894;1439195899.48 +2012/01/01,1681435126;1681435126;1681435126,1376019972.6;1462308891.32247;1548597810.05 +2012/02/01,1663482355;1663482355;1663482355,1494268594.88;1580557513.60023;1666846432.32 +2012/03/01,1858551326;1858551326;1858551326,1622079312.24;1708368230.96854;1794657149.69 +2012/04/01,,1760225357.07;1846514275.79798;1932803194.52 +2012/05/01,,1909542488.65;1995831407.37327;2082120326.1 +2012/06/01,,2070934049.37;2157222968.09006;2243511886.81 diff --git a/data/graphs/active_editors_target.json b/data/graphs/active_editors_target.json index 18ed591..70ba350 100644 --- a/data/graphs/active_editors_target.json +++ b/data/graphs/active_editors_target.json @@ -1 +1 @@ -{"options":{"animatedZooms":true,"avoidMinZero":false,"axis":null,"axisLabelColor":"#666666","axisLabelFontSize":11,"axisLabelFormatter":null,"axisLabelWidth":50,"axisLineColor":"#AAAAAA","axisLineWidth":0.3,"axisTickSize":3,"colorSaturation":1,"colorValue":0.5,"colors":["#FF0097","#EF8158","#83BB32","#182B53","#4596FF","#553DC9","#AD3238","#00FFBC","#F1D950"],"connectSeparatedPoints":false,"customBars":false,"dateWindow":null,"delimiter":",","digitsAfterDecimal":2,"displayAnnotations":false,"drawPoints":true,"drawXAxis":true,"drawXGrid":true,"drawYAxis":true,"drawYGrid":true,"errorBars":false,"file":null,"fillAlpha":0.15,"fillGraph":false,"fractions":false,"gridLineColor":"#D8D8D8","gridLineWidth":0.3,"hideOverlayOnMouseOut":true,"highlightCircleSize":4,"includeZero":false,"interactionModel":null,"isZoomedIgnoreProgrammaticZoom":false,"labels":null,"labelsDiv":null,"labelsDivStyles":null,"labelsDivWidth":250,"labelsKMB":true,"labelsKMG2":false,"labelsSeparateLines":true,"labelsShowZeroValues":true,"legend":"always","logscale":true,"maxNumberWidth":30,"panEdgeFraction":null,"pixelsPerLabel":null,"pixelsPerXLabel":null,"pixelsPerYLabel":null,"pointSize":1,"rangeSelectorHeight":40,"rangeSelectorPlotFillColor":"#A7B1C4","rangeSelectorPlotStrokeColor":"#808FAB","rightGap":20,"rollPeriod":1,"showLabelsOnHighlight":true,"showRangeSelector":false,"showRoller":false,"sigFigs":null,"sigma":2,"stackedGraph":false,"stepPlot":false,"strokePattern":null,"strokeWidth":3,"ticker":null,"title":null,"titleHeight":18,"valueFormatter":null,"valueRange":null,"visibility":null,"wilsonInterval":true,"xAxisHeight":null,"xAxisLabelFormatter":null,"xAxisLabelWidth":55,"xLabelHeight":18,"xValueFormatter":null,"xValueParser":null,"xlabel":null,"y2label":null,"yAxisLabelFormatter":null,"yAxisLabelWidth":50,"yLabelWidth":18,"yValueFormatter":null,"ylabel":null},"slug":"active_editors_target","name":"Active Editors - Target","desc":"","dataset":"/data/datasources/rc/rc_active_editors_target.csv","width":"auto","height":320,"chartType":"dygraphs","parents":["root"],"id":"active_editors_target"} \ No newline at end of file +{"options":{"animatedZooms":true,"avoidMinZero":false,"axis":null,"axisLabelColor":"#666666","axisLabelFontSize":11,"axisLabelFormatter":null,"axisLabelWidth":50,"axisLineColor":"#AAAAAA","axisLineWidth":0.3,"axisTickSize":3,"colorSaturation":1,"colorValue":0.5,"colors":["#FF0097","#EF8158","#83BB32","#182B53","#4596FF","#553DC9","#AD3238","#00FFBC","#F1D950"],"connectSeparatedPoints":false,"customBars":true,"dateWindow":null,"delimiter":",","digitsAfterDecimal":2,"displayAnnotations":false,"drawPoints":true,"drawXAxis":true,"drawXGrid":true,"drawYAxis":true,"drawYGrid":true,"errorBars":false,"file":null,"fillAlpha":0.15,"fillGraph":false,"fractions":false,"gridLineColor":"#D8D8D8","gridLineWidth":0.3,"hideOverlayOnMouseOut":true,"highlightCircleSize":4,"includeZero":false,"interactionModel":null,"isZoomedIgnoreProgrammaticZoom":false,"labels":null,"labelsDiv":null,"labelsDivStyles":null,"labelsDivWidth":250,"labelsKMB":true,"labelsKMG2":false,"labelsSeparateLines":true,"labelsShowZeroValues":true,"legend":"always","logscale":true,"maxNumberWidth":30,"panEdgeFraction":null,"pixelsPerLabel":null,"pixelsPerXLabel":null,"pixelsPerYLabel":null,"pointSize":1,"rangeSelectorHeight":40,"rangeSelectorPlotFillColor":"#A7B1C4","rangeSelectorPlotStrokeColor":"#808FAB","rightGap":20,"rollPeriod":1,"showLabelsOnHighlight":true,"showRangeSelector":false,"showRoller":false,"sigFigs":null,"sigma":2,"stackedGraph":false,"stepPlot":false,"strokePattern":null,"strokeWidth":3,"ticker":null,"title":null,"titleHeight":18,"valueFormatter":null,"valueRange":null,"visibility":null,"wilsonInterval":true,"xAxisHeight":null,"xAxisLabelFormatter":null,"xAxisLabelWidth":55,"xLabelHeight":18,"xValueFormatter":null,"xValueParser":null,"xlabel":null,"y2label":null,"yAxisLabelFormatter":null,"yAxisLabelWidth":50,"yLabelWidth":18,"yValueFormatter":null,"ylabel":null},"slug":"active_editors_target","name":"Active Editors - Target","desc":"","dataset":"/data/datasources/rc/rc_active_editors_target.csv","width":"auto","height":320,"chartType":"dygraphs","parents":["root"],"id":"active_editors_target"} \ No newline at end of file diff --git a/data/graphs/pageviews_mobile_target.json b/data/graphs/pageviews_mobile_target.json index 8530ced..1aa6238 100644 --- a/data/graphs/pageviews_mobile_target.json +++ b/data/graphs/pageviews_mobile_target.json @@ -1 +1 @@ -{"options":{"animatedZooms":true,"avoidMinZero":false,"axis":null,"axisLabelColor":"#666666","axisLabelFontSize":14,"axisLabelFormatter":null,"axisLabelWidth":50,"axisLineColor":"#AAAAAA","axisLineWidth":0.3,"axisTickSize":3,"colorSaturation":1,"colorValue":0.5,"colors":["#FF0097","#EF8158","#83BB32","#182B53","#4596FF","#553DC9","#AD3238","#00FFBC","#F1D950"],"connectSeparatedPoints":false,"customBars":false,"dateWindow":null,"delimiter":",","digitsAfterDecimal":2,"displayAnnotations":false,"drawPoints":true,"drawXAxis":true,"drawXGrid":true,"drawYAxis":true,"drawYGrid":true,"errorBars":false,"file":null,"fillAlpha":0.15,"fillGraph":false,"fractions":false,"gridLineColor":"#D8D8D8","gridLineWidth":0.3,"hideOverlayOnMouseOut":false,"highlightCircleSize":4,"includeZero":false,"interactionModel":null,"isZoomedIgnoreProgrammaticZoom":false,"labels":null,"labelsDiv":null,"labelsDivStyles":null,"labelsDivWidth":250,"labelsKMB":true,"labelsKMG2":false,"labelsSeparateLines":true,"labelsShowZeroValues":true,"legend":"always","logscale":false,"maxNumberWidth":30,"panEdgeFraction":null,"pixelsPerLabel":null,"pixelsPerXLabel":null,"pixelsPerYLabel":null,"pointSize":1,"rangeSelectorHeight":40,"rangeSelectorPlotFillColor":"#A7B1C4","rangeSelectorPlotStrokeColor":"#808FAB","rightGap":20,"rollPeriod":1,"showLabelsOnHighlight":true,"showRangeSelector":false,"showRoller":false,"sigFigs":null,"sigma":2,"stackedGraph":false,"stepPlot":false,"strokePattern":null,"strokeWidth":3,"ticker":null,"title":null,"titleHeight":18,"valueFormatter":null,"valueRange":null,"visibility":null,"wilsonInterval":true,"xAxisHeight":null,"xAxisLabelFormatter":null,"xAxisLabelWidth":55,"xLabelHeight":18,"xValueFormatter":null,"xValueParser":null,"xlabel":null,"y2label":null,"yAxisLabelFormatter":null,"yAxisLabelWidth":50,"yLabelWidth":18,"yValueFormatter":null,"ylabel":null},"slug":"pageviews_mobile_target","name":"Page Views to Mobile Site - Target","desc":"","dataset":"/data/datasources/rc/rc_page_requests_mobile_target.csv","width":"auto","height":320,"chartType":"dygraphs","parents":["root"],"id":"pageviews_mobile_target"} \ No newline at end of file +{"options":{"animatedZooms":true,"avoidMinZero":false,"axis":null,"axisLabelColor":"#666666","axisLabelFontSize":14,"axisLabelFormatter":null,"axisLabelWidth":50,"axisLineColor":"#AAAAAA","axisLineWidth":0.3,"axisTickSize":3,"colorSaturation":1,"colorValue":0.5,"colors":["#FF0097","#EF8158","#83BB32","#182B53","#4596FF","#553DC9","#AD3238","#00FFBC","#F1D950"],"connectSeparatedPoints":false,"customBars":true,"dateWindow":null,"delimiter":",","digitsAfterDecimal":2,"displayAnnotations":false,"drawPoints":true,"drawXAxis":true,"drawXGrid":true,"drawYAxis":true,"drawYGrid":true,"errorBars":false,"file":null,"fillAlpha":0.15,"fillGraph":false,"fractions":false,"gridLineColor":"#D8D8D8","gridLineWidth":0.3,"hideOverlayOnMouseOut":false,"highlightCircleSize":4,"includeZero":false,"interactionModel":null,"isZoomedIgnoreProgrammaticZoom":false,"labels":null,"labelsDiv":null,"labelsDivStyles":null,"labelsDivWidth":250,"labelsKMB":true,"labelsKMG2":false,"labelsSeparateLines":true,"labelsShowZeroValues":true,"legend":"always","logscale":false,"maxNumberWidth":30,"panEdgeFraction":null,"pixelsPerLabel":null,"pixelsPerXLabel":null,"pixelsPerYLabel":null,"pointSize":1,"rangeSelectorHeight":40,"rangeSelectorPlotFillColor":"#A7B1C4","rangeSelectorPlotStrokeColor":"#808FAB","rightGap":20,"rollPeriod":1,"showLabelsOnHighlight":true,"showRangeSelector":false,"showRoller":false,"sigFigs":null,"sigma":2,"stackedGraph":false,"stepPlot":false,"strokePattern":null,"strokeWidth":3,"ticker":null,"title":null,"titleHeight":18,"valueFormatter":null,"valueRange":null,"visibility":null,"wilsonInterval":true,"xAxisHeight":null,"xAxisLabelFormatter":null,"xAxisLabelWidth":55,"xLabelHeight":18,"xValueFormatter":null,"xValueParser":null,"xlabel":null,"y2label":null,"yAxisLabelFormatter":null,"yAxisLabelWidth":50,"yLabelWidth":18,"yValueFormatter":null,"ylabel":null},"slug":"pageviews_mobile_target","name":"Page Views to Mobile Site - Target","desc":"","dataset":"/data/datasources/rc/rc_page_requests_mobile_target.csv","width":"auto","height":320,"chartType":"dygraphs","parents":["root"],"id":"pageviews_mobile_target"} \ No newline at end of file diff --git a/msc/old_rc_new/new_charts.py b/msc/old_rc_new/new_charts.py index 9d196ec..35bfa21 100644 --- a/msc/old_rc_new/new_charts.py +++ b/msc/old_rc_new/new_charts.py @@ -5,7 +5,6 @@ import os from collections import OrderedDict import parse_csv as p -import targets def mobile_page_requests(): @@ -46,5 +45,3 @@ def mobile_page_requests(): def create_all(): mobile_page_requests() - - targets.create_all() \ No newline at end of file diff --git a/msc/old_rc_new/old_to_new.py b/msc/old_rc_new/old_to_new.py index cee76b9..9da83d5 100644 --- a/msc/old_rc_new/old_to_new.py +++ b/msc/old_rc_new/old_to_new.py @@ -16,13 +16,21 @@ output_folder = './output' import old_charts import new_charts import comscore - +import targets def create_all(): + # recreate ErikZ charts old_charts.create_all() - new_charts.create_all() comscore.create_all() + # new charts + new_charts.create_all() + + # target charts + # TODO: start/end date for projection are hard-coded in target.py + # Note that it used previously deployed data to /data/datasources/rc to create datafiles. + targets.create_all() + if __name__ == '__main__': create_all() diff --git a/msc/old_rc_new/targets.py b/msc/old_rc_new/targets.py index 3cad0b0..8a3d003 100644 --- a/msc/old_rc_new/targets.py +++ b/msc/old_rc_new/targets.py @@ -28,53 +28,87 @@ def extractData(fn, sep,format,fields): return data -def dygraphOutput(out_fn,tss,actual,target,a_names,t_names): +def dygraphOutput(out_fn,tss,actual,target,a_names,t_names,error_bar=0,error_bar_names=['Target']): + '''Saves a valid dygraph file in `out_fn`. + + :arg actual: dict containing actual data points (can be multiple keys) + :arg target: dict containing target data points (can be multiple keys) + :arg a_names: ordered list of actual labels (have to correspond to keys in `actual`) + :arg t_names: ordered list of target labels (have to correspond to keys in `target`) + :arg t_names: optional int, width of error bar as percentage of max target['Target'] value + ''' with open(os.path.join(old_to_new.output_folder,out_fn),'w') as out: headers = ["Date"]+a_names+t_names #[f[0] for f in a_fields]+[f[0] for f in t_fields] out.write(','.join(headers)+'\n') + + if error_bar: + + maxTarget = max([float(a) for n in error_bar_names for a in target[n].values() if a!='']) + delta = maxTarget/100*error_bar + + + # iterate ordered list of timestamps for ts in tss: - #ordered list of timestamps - # print actual['Total'] - vals = [utils.dygraph_date(ts)]+[actual[n][ts] for n in a_names]+[target[n][ts] for n in t_names] + + if error_bar: + # show error bar of target_error_bar percent for target line + + vals = [utils.dygraph_date(ts)] + for n in a_names: + if actual[n][ts] != '': + vals += [';'.join(3*[actual[n][ts]])] + else: + vals += [''] + + for n in t_names: + if target[n][ts] != '': + if n in error_bar_names: + vals += ['%s;%s;%s'%(float(target[n][ts])-delta,target[n][ts],float(target[n][ts])+delta)] + else: + vals += [';'.join(3*[target[n][ts]])] + else: + vals += [''] + + # vals += ['%s;%s;%s'%(int(target[n][ts])-delta,target[n][ts],int(target[n][ts])+delta) for n in t_names] + + else: + vals = [utils.dygraph_date(ts)]+[actual[n][ts] for n in a_names]+[target[n][ts] for n in t_names] + + # print ','.join(vals) out.write(','.join(vals)+'\n') -def active_editors_target(): +def active_editors_target(min_ts,max_ts): # input files - metricfile = '../../data/datasources/rc/rc_active_editors_count.csv' + + metricfile = os.path.join(old_to_new.output_folder,'rc_active_editors_count.csv') + # metricfile = '../../data/datasources/rc/rc_active_editors_count.csv' targetfile = '../../data/targets/editor_target.csv' # time range - # TODO, extract automatically - min_ts = '200903' - max_ts = '201206' tss,tss_i = utils.create_time_stamps_month(fromym=min_ts,toym=max_ts) - target = extractData(targetfile, '\t','MMM-YY',[('Projected',1),('Target',2)]) actual = extractData(metricfile, ',','YYYY/MM/DD',[('Total',1)]) - dygraphOutput('rc_active_editors_target.csv',tss,actual,target,['Total'],['Target','Projected']) + dygraphOutput('rc_active_editors_target.csv',tss,actual,target,['Total'],['Target','Projected'],error_bar=2)#,error_bar_names=['Target','Projected']) -def mobile_pageviews_target(): +def mobile_pageviews_target(min_ts,max_ts): # input files metricfile = '../../data/datasources/rc/rc_page_requests_mobile.csv' targetfile = '../../data/targets/mobile_target.csv' # time range - # TODO, extract automatically - min_ts = '201006' - max_ts = '201206' tss,tss_i = utils.create_time_stamps_month(fromym=min_ts,toym=max_ts) - target = extractData(targetfile, '\t','MMM-YY',[('Target',3)]) actual = extractData(metricfile, ',','YYYY/MM/DD',[('Total',1)]) - dygraphOutput('rc_page_requests_mobile_target.csv',tss,actual,target,['Total'],['Target']) + dygraphOutput('rc_page_requests_mobile_target.csv',tss,actual,target,['Total'],['Target'],error_bar=4) def create_all(): - active_editors_target() - mobile_pageviews_target() + + active_editors_target(min_ts='200903',max_ts='201206') + mobile_pageviews_target(min_ts='201006',max_ts='201206') diff --git a/msc/old_rc_new/utils.py b/msc/old_rc_new/utils.py index b4a13aa..02b9f61 100644 --- a/msc/old_rc_new/utils.py +++ b/msc/old_rc_new/utils.py @@ -9,174 +9,174 @@ months_dict = {'Mar': '03', 'Feb': '02', 'Aug': '08', 'Sep': '09', 'Apr': '04', def fix_date(t,format): - '''Formats the date from Jan/2001 into 2001/01/01 style''' - if format == 'MMM-YY': - return '20%s/%s/01'%(t[4:], months_dict[t[:3]]) - elif format == 'MM/YYYY': - return '%s/%s/01'%(t[3:],t[:2]) - elif format == 'YYYY-MM': - return '%s/%s/01'%(t[:4],t[5:]) + '''Formats the date from Jan/2001 into 2001/01/01 style''' + if format == 'MMM-YY': + return '20%s/%s/01'%(t[4:], months_dict[t[:3]]) + elif format == 'MM/YYYY': + return '%s/%s/01'%(t[3:],t[:2]) + elif format == 'YYYY-MM': + return '%s/%s/01'%(t[:4],t[5:]) def fix_dates(dates,format): - return [fix_date(t,format) for t in dates] - + return [fix_date(t,format) for t in dates] + def dygraph_date(ts): - return '%s/%s/01'%(ts[:4],ts[4:]) + return '%s/%s/01'%(ts[:4],ts[4:]) def to_ts(date,format): - '''Formats the date from Jan/2001 or simple ts style''' - if format != 'YYYY/MM/DD': - date = fix_date(date,format) - return '%s%s'%(date[:4],date[5:7]) + '''Formats the date from Jan/2001 or simple ts style''' + if format != 'YYYY/MM/DD': + date = fix_date(date,format) + return '%s%s'%(date[:4],date[5:7]) def create_time_stamps_month(fromym='200101',toym='201012'): - ''' - Helper data structures for time stamps - List of all time unites, i.e. every month. yyyymm - ''' - time_stamps = [] - # A dictionary that serves as a lookup for the index of atime stamp - time_stamps_index = {} - - min_year = fromym[:-2] - min_month = fromym[-2:] - max_year = toym[:-2] - max_month = toym[-2:] - - i = 0 - #add remaining months in the first year - for m in range(int(min_month),13): - ts = '%s%02d'%(min_year,m) - time_stamps.append(ts) - time_stamps_index[ts] = i - i += 1 - #add intermediate years - for y in range(int(min_year)+1,int(max_year)): - for m in range(1,13): - ts = '%s%02d'%(y,m) - time_stamps.append(ts) - time_stamps_index[ts] = i - i += 1 - #last year - for m in range(1,int(max_month)+1): - ts = '%s%02d'%(max_year,m) - time_stamps.append(ts) - time_stamps_index[ts] = i - i += 1 - - return (time_stamps,time_stamps_index) + ''' + Helper data structures for time stamps + List of all time unites, i.e. every month. yyyymm + ''' + time_stamps = [] + # A dictionary that serves as a lookup for the index of atime stamp + time_stamps_index = {} + + min_year = fromym[:-2] + min_month = fromym[-2:] + max_year = toym[:-2] + max_month = toym[-2:] + + i = 0 + #add remaining months in the first year + for m in range(int(min_month),13): + ts = '%s%02d'%(min_year,m) + time_stamps.append(ts) + time_stamps_index[ts] = i + i += 1 + #add intermediate years + for y in range(int(min_year)+1,int(max_year)): + for m in range(1,13): + ts = '%s%02d'%(y,m) + time_stamps.append(ts) + time_stamps_index[ts] = i + i += 1 + #last year + for m in range(1,int(max_month)+1): + ts = '%s%02d'%(max_year,m) + time_stamps.append(ts) + time_stamps_index[ts] = i + i += 1 + + return (time_stamps,time_stamps_index) def create_time_stamps_day(fromymd='20010101',toymd='20101231'): - ''' - Helper data structures for time stamps - List of all time unites, i.e. every month. yyyymm - ''' - - import calendar - - time_stamps = [] - # A dictionary that serves as a lookup for the index of atime stamp - time_stamps_index = {} - - min_year = int(fromymd[:4]) - min_month = int(fromymd[4:6]) - min_day = int(fromymd[6:8]) - max_year = int(toymd[:4]) - max_month = int(toymd[4:6]) - max_day = int(toymd[6:8]) - i = 0 - #add remaining months in the first year - for m in range(min_month,13): - # calender.monthrange returns a tuple (day of the week, number of days in the month) - nds = calendar.monthrange(min_year,m)[1] - for d in range(min_day,nds+1): - ts = '%s%02d%02d'%(min_year,m,d) - time_stamps.append(ts) - time_stamps_index[ts] = i - i += 1 - #add intermediate years - for y in range(min_year+1,max_year): - for m in range(1,13): - # calender.monthrange returns a tuple (day of the week, number of days in the month) - nds = calendar.monthrange(y,m)[1] - for d in range(1,nds+1): - ts = '%s%02d%02d'%(y,m,d) - time_stamps.append(ts) - time_stamps_index[ts] = i - i += 1 - #to last month - for m in range(1,max_month): - nds = calendar.monthrange(max_year,m)[1] - for d in range(1,nds+1): - ts = '%s%02d%02d'%(max_year,m,d) - time_stamps.append(ts) - time_stamps_index[ts] = i - i += 1 - #to last day - for d in range(1,max_day+1): - ts = '%s%02d%02d'%(max_year,max_month,d) - time_stamps.append(ts) - time_stamps_index[ts] = i - i += 1 - - - return (time_stamps,time_stamps_index) - - + ''' + Helper data structures for time stamps + List of all time unites, i.e. every month. yyyymm + ''' + + import calendar + + time_stamps = [] + # A dictionary that serves as a lookup for the index of atime stamp + time_stamps_index = {} + + min_year = int(fromymd[:4]) + min_month = int(fromymd[4:6]) + min_day = int(fromymd[6:8]) + max_year = int(toymd[:4]) + max_month = int(toymd[4:6]) + max_day = int(toymd[6:8]) + i = 0 + #add remaining months in the first year + for m in range(min_month,13): + # calender.monthrange returns a tuple (day of the week, number of days in the month) + nds = calendar.monthrange(min_year,m)[1] + for d in range(min_day,nds+1): + ts = '%s%02d%02d'%(min_year,m,d) + time_stamps.append(ts) + time_stamps_index[ts] = i + i += 1 + #add intermediate years + for y in range(min_year+1,max_year): + for m in range(1,13): + # calender.monthrange returns a tuple (day of the week, number of days in the month) + nds = calendar.monthrange(y,m)[1] + for d in range(1,nds+1): + ts = '%s%02d%02d'%(y,m,d) + time_stamps.append(ts) + time_stamps_index[ts] = i + i += 1 + #to last month + for m in range(1,max_month): + nds = calendar.monthrange(max_year,m)[1] + for d in range(1,nds+1): + ts = '%s%02d%02d'%(max_year,m,d) + time_stamps.append(ts) + time_stamps_index[ts] = i + i += 1 + #to last day + for d in range(1,max_day+1): + ts = '%s%02d%02d'%(max_year,max_month,d) + time_stamps.append(ts) + time_stamps_index[ts] = i + i += 1 + + + return (time_stamps,time_stamps_index) + + def computeMonthStartEndtime(ym): - ''' - Returns the starting and end datetime object for the yyyymm passed. I.e. the first and last day of the month + ''' + Returns the starting and end datetime object for the yyyymm passed. I.e. the first and last day of the month - :arg ym: str, 'yyyymm' format - :returns: tuple of datetime objects - ''' - from datetime import datetime - import calendar + :arg ym: str, 'yyyymm' format + :returns: tuple of datetime objects + ''' + from datetime import datetime + import calendar - - y = int(ym[:4]) - m = int(ym[4:]) + + y = int(ym[:4]) + m = int(ym[4:]) - start = datetime(y, m, 1) - # days in the given month - # calender.monthrange returns a tuple (day of the week, number of days in the month) - d = calendar.monthrange(y,m)[1] + start = datetime(y, m, 1) + # days in the given month + # calender.monthrange returns a tuple (day of the week, number of days in the month) + d = calendar.monthrange(y,m)[1] - end = datetime(y, m, d) + end = datetime(y, m, d) - return (start,end) + return (start,end) def numberOfMonths(ymStart,ymEnd): - '''Returns the number of months between the parameters. + '''Returns the number of months between the parameters. - :arg ymStart: str, 'yyyymm' format - :arg ymEnd: str, 'yyyymm' format - :returns: int, number of month - ''' - months = 0 - months += (int(ymEnd[:4])-(int(ymStart[:4])+1)) * 12 - if int(ymEnd[:4])==int(ymStart[:4]): - #same year - months += (int(ymEnd[4:])-int(ymStart[4:]))+1 - else: - months += 12-int(ymStart[4:])+1 - months += int(ymEnd[4:]) + :arg ymStart: str, 'yyyymm' format + :arg ymEnd: str, 'yyyymm' format + :returns: int, number of month + ''' + months = 0 + months += (int(ymEnd[:4])-(int(ymStart[:4])+1)) * 12 + if int(ymEnd[:4])==int(ymStart[:4]): + #same year + months += (int(ymEnd[4:])-int(ymStart[4:]))+1 + else: + months += 12-int(ymStart[4:])+1 + months += int(ymEnd[4:]) - return months + return months def movingAverage(array, WINDOW=5): - try: - import numpy as N - except: - logging.warning("Moving average can't be computed (Numpy not installed)") - return - - - weightings = N.repeat(1.0, WINDOW) / WINDOW - return N.convolve(array, weightings)[WINDOW-1:-(WINDOW-1)] + try: + import numpy as N + except: + logging.warning("Moving average can't be computed (Numpy not installed)") + return + + + weightings = N.repeat(1.0, WINDOW) / WINDOW + return N.convolve(array, weightings)[WINDOW-1:-(WINDOW-1)] -- 1.7.0.4