diff --git a/rivretrieve/cached_site_data/uk_sepa_sites.csv b/rivretrieve/cached_site_data/uk_sepa_sites.csv new file mode 100644 index 0000000..1144a09 --- /dev/null +++ b/rivretrieve/cached_site_data/uk_sepa_sites.csv @@ -0,0 +1,395 @@ +gauge_id,station_name,station_no,latitude,longitude +36744,Aberuthven,14969,56.31968188,-3.658685969 +37602,Abington,322551,55.48691533,-3.69056827 +37582,Aboyne,302205,57.07726163,-2.771636889 +36258,Airyhemming,133141,54.90049986,-4.839842855 +37182,Alford,234170,57.24230294,-2.717123004 +37138,Allnabad,234151,58.34717676,-4.644621014 +37970,Allt a' Ghlas-charnaich,368538,57.70934072,-4.836082499 +36688,Almondbank,14954,56.41527516,-3.51359839 +36404,Almondell,14869,55.90181846,-3.463017734 +37144,Alness,234152,57.69601068,-4.258760757 +36834,Ancrum,15002,55.51244139,-2.580776051 +36462,Anie,14888,56.25788155,-4.285414699 +37148,Apigill,234153,58.48007214,-4.207749187 +36574,Arbroath,14922,56.56723031,-2.588171626 +37156,Ardachy Bridge,234155,57.12952381,-4.678200965 +37160,Ariundle,234157,56.70722202,-5.556782552 +37998,Athelstaneford,372871,55.9852479,-2.736326688 +36008,Auchendrane,133060,55.40997977,-4.627054661 +36012,Auchengeich,133061,55.91939055,-4.114140851 +37164,Avochie,234161,57.50564634,-2.782377895 +36486,Balfour Mains,14894,56.1847332,-3.08157055 +38270,Ballachraggan,516803,57.52824441,-3.451567365 +36630,Ballathie,14937,56.5144141,-3.386963945 +36564,Balmossie,14919,56.48175497,-2.85166534 +37170,Balnaan,234163,57.30214571,-3.699157201 +37128,Balnafoich,206846,57.38787077,-4.188250573 +36016,Balnowlart,133062,55.10717871,-4.968322113 +36494,Bannockburn,14897,56.09528478,-3.911816854 +36020,Barnaline Lodge,133063,56.27473893,-5.278048706 +36266,Beattock,133143,55.31167269,-3.456710814 +36024,Blairston,133064,55.79734306,-4.068305537 +37174,Boat o Brig,234167,57.55104115,-3.140387991 +37178,Boat of Garten,234168,57.25078305,-3.746867237 +38340,Boleside,533744,55.59164323,-2.798878774 +36450,Bonnybridge,14885,56.00182143,-3.886990553 +37562,Borgie Bridge,234334,58.49740038,-4.279784356 +36028,Braevallich,133065,56.21639554,-5.296246434 +36556,Brechin,14917,56.72692276,-2.655900389 +36466,Bridge of Allan,14889,56.15901974,-3.956367475 +36458,Bridge of Teith,14887,56.18573476,-4.055574525 +36276,Bridgemuir,133145,55.14632871,-3.428225271 +36032,Bridgend,133066,55.94910632,-4.129006198 +37598,Brighouse Burn,321630,54.78864247,-4.117404209 +36804,Brockhoperig,14987,55.4061286,-3.211513384 +37192,Brodie,234173,57.59244808,-3.708878989 +36880,Broxmouth,15029,55.99077195,-2.48676507 +37196,Bruachrobie,234174,58.0105569,-3.877565412 +36280,Brydekirk,133146,55.02165102,-3.267726412 +37950,Burrelton Burn,368492,56.5211801,-3.294527587 +37200,Cabrach,234176,57.34816,-3.034565786 +36778,Cademuir,14977,55.61926614,-3.244232636 +36036,Calderpark,133069,55.83754619,-4.10777909 +37206,Camisky,234178,56.87846936,-5.046521526 +36040,Candermill,133070,55.70088757,-3.966908369 +36284,Canonbie,133148,55.0662645,-2.946231369 +36288,Capenoch,133149,55.22736754,-3.817044727 +36626,Caputh,14936,56.53919737,-3.492942476 +37594,Carron (Stonehaven),320374,56.96221164,-2.216208291 +38444,Carron upstream (Stonehaven),565197,56.96155865,-2.219756428 +38210,Castle O'er,506142,55.22290808,-3.180030664 +36524,Castlehill Reservoir,14908,56.2092222,-3.60889654 +36044,Catrine,133071,55.50462561,-4.336590474 +35982,Caulkerbush,116013,54.89874408,-3.672536736 +38390,Chapeltoun Gauging Station,541851,55.6639264,-4.554890682 +37210,Charr,234183,56.94084906,-2.619402267 +36838,Chesters,15004,55.37862961,-2.589607681 +35978,Claggan,116011,56.82151065,-5.088839432 +36858,Coldstream,15013,55.64977661,-2.256597919 +36428,Colinton,14880,55.90575791,-3.262141625 +36568,Colliston,14921,56.61000991,-2.637841019 +36674,Comrie Bridge,14951,56.61422921,-3.980864711 +36048,Condorrat,133072,55.92951508,-4.019603678 +38070,Corrievuic,472872,57.51349507,-5.001875565 +38480,Cowbridge,5981,55.89709776,-3.05890347 +279917,Craig Douglas,14983,55.5086,-3.1265 +36052,Craigend,133073,55.86450249,-4.534803989 +37876,Craigendarroch,356326,56.56990238,-5.749845756 +38042,Craigforth,4638,56.1360998,-3.970218251 +36698,Craighall (Perth),14956,56.60905796,-3.347175751 +36396,Craigiehall,14867,55.96313343,-3.338532726 +37236,Creed Bridge,234190,58.20454028,-6.424227928 +38336,Crimmonmogate,533741,57.61852769,-1.928524204 +37242,Culfork,234191,57.18022427,-3.11016313 +37246,Cullen,234192,57.68712295,-2.83323079 +36726,Cultybraggan,14965,56.35972783,-4.001442447 +38382,Cunninghamhead,538826,55.64794589,-4.584018331 +36292,Dalbeattie,133151,54.92987759,-3.840969588 +36056,Daldowie,133074,55.82998694,-4.121948322 +36296,Dalgig,133152,55.38782272,-4.277163013 +36586,Dalhousie Bridge,14925,56.8021744,-2.67047775 +36416,Dalmore,14873,55.84190258,-3.195905294 +35986,Dalry,116020,55.70198453,-4.716787117 +36536,Damleys Cottage,14912,56.22128534,-3.361631179 +36300,Dargall Lane,133153,55.0785507,-4.428445527 +36650,Dean Bridge,14946,56.5988796,-3.153071161 +36800,Deephope,14986,55.41241422,-3.141967218 +37252,Delnashaugh,234198,57.39970792,-3.357845061 +37548,Dionard,234330,58.49807306,-4.821196451 +36060,Dippen,133077,55.58274848,-5.496523588 +37258,Diriebught House,234200,57.47804173,-4.207153032 +37262,Dosmucheran,234202,57.59587318,-5.005770119 +36386,Dow Craig,133183,55.37721095,-4.306229017 +36066,Dreghorn,133078,55.61141199,-4.619377401 +36304,Drumlanrig,133154,55.27462204,-3.799524299 +36510,Duchray,14904,56.15649685,-4.468109175 +38402,Duchray Water,554833,56.19275854,-4.564061145 +37268,Dunphail,234206,57.51151031,-3.661788536 +37272,Durkadale,234207,59.10852817,-3.232172832 +36822,Earlston,14997,55.64075254,-2.691879301 +36074,Eas Daimh,133081,56.40771881,-4.855886468 +36408,East Linton,14870,55.98249658,-2.657198972 +37278,Easter Turnaig,234209,57.96185041,-4.700859639 +36078,Eglinton Castle,133083,55.64280886,-4.677656958 +37282,Ellon,234210,57.36377128,-2.090017282 +37962,Elphin,368495,58.05185192,-5.03101345 +36516,Elrig,14905,56.15679797,-4.46563232 +36866,Eyemouth Mill,15017,55.8651192,-2.095026904 +38014,Faich Bhiorach,378262,57.69020841,-5.660695458 +36082,Fairholm,133084,55.74419908,-3.986819701 +37286,Fasnakyle,234215,57.31811088,-4.799267396 +36490,Fauld Mill,14895,56.11140757,-3.748668124 +37290,Feshie Bridge,234217,57.11889938,-3.901082854 +36308,Fiddlers Ford,133155,55.09849668,-3.680572053 +37294,Firhall,234218,57.57160708,-3.869783835 +37298,Forres,234221,57.60477145,-3.644084919 +36748,Forteviot Bridge,14970,56.34759942,-3.548359476 +36312,Friars Carse,133156,55.14868484,-3.690747738 +36086,Gaidrew,133086,56.04809931,-4.434505213 +38500,Galashiels,9683,55.62706185,-2.827709454 +36480,Gargunnock,14892,56.13266077,-4.070571402 +37302,Garve,234225,57.61669586,-4.688028816 +36550,Gella Bridge,14916,56.77535832,-3.027929218 +36090,Glen Orchy,133087,56.44657591,-4.852749436 +36094,Glen Strae,133088,56.42522107,-5.000795867 +37940,Glenbreck,368490,55.48202196,-3.477754728 +36100,Glenfalloch,133089,56.33773623,-4.717635246 +37648,Glenfield,83013,55.60118281,-4.493559809 +36316,Glenlochar,133157,54.95554151,-3.979143444 +37306,Glenmeanie,234227,57.53261438,-4.863772899 +36474,Glenochil,14891,56.14364525,-3.839304837 +36790,Gordon Arms,14984,55.51137227,-3.096406906 +37310,Grange,234228,57.54279921,-2.847774986 +37314,Grantown,234229,57.32177064,-3.606725707 +38290,Graystone Bridge,518372,56.0675405,-4.282616591 +36320,Green Burn,133158,55.0828864,-4.381888551 +36582,Guthrie,14924,56.64274771,-2.706028939 +37318,Halkirk,234231,58.5154974,-3.493004244 +36330,Hall Bridge,133160,55.39297897,-4.079109518 +37324,Halladale,234232,58.47881083,-3.90301916 +36104,Happendon,133092,55.5801981,-3.816796943 +37328,Haughton,234235,57.27134676,-2.404963889 +36830,Hawick,15001,55.43447121,-2.756965079 +36108,Hawkhead,133093,55.83606531,-4.398537371 +36112,Hazelbank,133094,55.68735585,-3.854153414 +36454,Headswood,14886,56.01500087,-3.875456793 +38492,Hermitage,8402,56.5609809,-3.607201681 +37332,Heugh Head,234237,57.02524569,-2.517160888 +36116,Hoodston Bridge,133096,55.60122684,-4.428249956 +37606,Howard Park,322555,55.60465827,-4.499004047 +37974,Huntly GS,369548,57.45498575,-2.807560217 +279903,Hutton Castle,15020,55.7889,-2.1917 +37340,Inverbervie,234241,56.85144056,-2.286119413 +37344,Inverbroom,234242,57.81047099,-5.058237756 +37348,Invergairn,234243,57.05992614,-3.069535364 +37988,Inveriscandye,371579,56.79987798,-2.626620926 +36120,Inverlochy,133097,56.40496092,-4.923669812 +36590,Invermark,14926,56.91121364,-2.914811966 +37352,Invermuick,234247,57.03951544,-3.049304978 +37356,Invertruim,234248,57.04053804,-4.164639706 +37360,Inverugie,234249,57.52669115,-1.832792272 +36842,Jedburgh,15006,55.48529122,-2.54668704 +38328,Kelly Den - Arbilot,532820,56.55248062,-2.640451372 +36604,Kemback,14929,56.3309034,-2.948442952 +38488,Kenmore,8295,56.59638815,-3.985061725 +37364,Kerrow Wood,234251,57.34937628,-4.736427892 +38152,Kilbirnie,494257,55.75642821,-4.685378197 +36124,Killermont,133099,55.90623638,-4.307622592 +36702,Killiecrankie,14958,56.75025399,-3.79910265 +37368,Killilan,234252,57.31881832,-5.424835603 +36616,Killin,14933,56.45816211,-4.331876561 +37374,Kilphedir,234253,58.14138911,-3.702814019 +36128,Kilwinning,133101,55.64563956,-4.692060096 +36470,Kinbuck,14890,56.2255551,-3.949530685 +36692,Kindrogan,14955,56.74926972,-3.544931301 +36756,Kingledores,14972,55.54240926,-3.413947163 +36738,Kinkell Bridge,14968,56.33018709,-3.726992915 +38066,Kinlochewe Lodge,471362,57.60438272,-5.291673059 +37384,Kinrara,234257,57.15134199,-3.85017621 +36506,Kinross,14903,56.19853589,-3.422943515 +36132,Kirkmuirhill,133102,55.66506695,-3.893674093 +36578,Kirkton Mill,14923,56.63544717,-2.564584008 +37934,Ladyflat Farm,368486,55.75636712,-2.369501202 +38018,Lagganbeg,396186,56.05128388,-4.52729393 +36136,Langholm A,133103,55.46679973,-4.362070221 +38222,Langholm Bridge,506155,55.1521906,-3.000489189 +37844,Langwell Lodge,354621,57.9748238,-5.100589967 +36500,Lathro,14902,56.2213192,-3.42872292 +37870,Laxdale,356217,57.86865846,-6.893581696 +37388,Laxford Bridge,234260,58.37221983,-4.991457712 +36140,Lennox Castle,133104,55.98029455,-4.220982816 +36420,Lennoxlove,14874,55.93584934,-2.784195821 +36144,Letham,133105,55.76643914,-4.284272139 +36424,Liberton,14876,55.92458009,-3.165787507 +38394,Limmerhaugh,542772,55.51674252,-4.189197017 +36810,Lindean,14990,55.57486097,-2.816174621 +36152,Linnbrane,133107,55.98877125,-4.57598193 +37396,Little Assynt,234264,58.17438109,-5.152302768 +37402,Littlemill,234265,57.04544067,-3.118285636 +38090,Loakmill No 2,474477,56.49180756,-3.508004624 +37838,Loch Ailsh Level,354613,58.04827715,-4.855312073 +38074,Loch Beannacharain,473090,57.5217999,-4.95927599 +36334,Loch Dee Outlet,133163,55.08809918,-4.386053697 +459053,Loch Katrine,14910,56.22701614,-4.43624573 +38098,Loch Leven Sluices,477620,56.17937821,-3.335224928 +38448,Logie Bridge,567430,57.49449251,-3.736619229 +36596,Logie Mill,14927,56.7668773,-2.493365398 +36338,Low Malzie,133167,54.85901864,-4.522077924 +35956,Lower Monachyle,116006,56.37472206,-4.4712587 +36390,Luffness,14865,56.02022484,-2.821888561 +36638,Luncarty,14940,56.46394135,-3.477636063 +36156,Luss,133108,56.10033657,-4.644123033 +36600,Luther Bridge,14928,56.7936908,-2.55997015 +36760,Lyne Ford,14973,55.64466586,-3.263056512 +36764,Lyne Station,14974,55.64733183,-3.25808918 +36160,Maidencots,133110,55.51579546,-3.695370189 +36164,Mainholm,133111,55.46104476,-4.592546164 +37414,Mar Lodge,234274,56.98804049,-3.486197825 +36706,Marble Lodge,14959,56.82242289,-3.816539353 +36646,Mill Bank,14945,56.54503404,-3.333677733 +37856,Mill Croft,354623,57.2886568,-7.368331541 +37958,Mill of Keithfield,368494,57.39549598,-2.268170959 +37418,Mill of Tore,234277,57.33389585,-4.575831167 +36168,Milliken Park,133112,55.82523457,-4.537788828 +36172,Milngavie,133113,55.93341456,-4.309062631 +36520,Milton,14907,56.18112306,-4.412580797 +36176,Milton of Campsie,133114,55.95994689,-4.153697207 +37422,Minmore,234279,57.34451357,-3.328222906 +36342,Minnoch Bridge,133168,55.03887143,-4.563503023 +36180,Monyquil Farm,133115,55.56419458,-5.271931931 +36346,Mossknowe,133169,55.01280562,-3.119586538 +36874,Mouthbridge,15022,55.77012815,-2.279193263 +37426,Moy Bridge,234284,57.55657789,-4.539965573 +37430,Muiresk,234286,57.53766592,-2.493533301 +37992,Murray Burn,372859,55.91287677,-3.317681386 +36400,Murrayfield,14868,55.94563152,-3.237786767 +37700,Murton,335650,56.65077608,-2.823539396 +38484,Musselburgh,635,55.93950941,-3.060909073 +37434,Ness-side,234287,57.4545366,-4.259846808 +37438,New Kelso,234289,57.42953514,-5.429999168 +36350,Newcastleton,133170,55.17647238,-2.81316152 +37444,Newhall Bridge,234290,57.65781737,-4.17662641 +36438,Newliston,14882,55.9429625,-3.420216359 +36186,Newmilns,133117,55.60610732,-4.331033075 +36684,Newton Bridge,14953,56.4611514,-3.799234379 +36354,Newton Stewart,133171,54.95726338,-4.480525846 +38496,Norham,9514,55.72229365,-2.163435109 +36848,Ormiston Mill,15007,55.54491924,-2.473314492 +36190,Overlee,133118,55.78987786,-4.266853356 +36194,Oxgang,133119,55.9354055,-4.13808503 +37448,Park,234291,57.075424,-2.335825098 +37452,Parkhill,234292,57.21832146,-2.187586406 +36782,Peebles,14979,55.64758358,-3.180313467 +36796,Philiphaugh,14985,55.53980549,-2.890068293 +37456,Pitcaple,234293,57.32351879,-2.464869508 +36716,Pitlochry,14963,56.69656771,-3.720794881 +36622,Pitnacree,14935,56.66006824,-3.756901303 +38110,Pluscarden Abbey,486057,57.59946107,-3.433170424 +37460,Polhollick,234294,57.05514629,-3.083180673 +37466,Polloch,234295,56.75844885,-5.613663683 +36446,Polmonthill,14884,55.99837265,-3.681998769 +37472,Poolewe,234296,57.76144201,-5.599076889 +36546,Prosen Bridge,14915,56.71469976,-2.988763564 +38278,Quinloch Bridge,517413,56.00926669,-4.375847831 +37480,Redcraig,234298,57.42475756,-2.802015318 +36358,Redhall,133175,55.16743102,-3.450205295 +36198,Redlees,133122,55.81909284,-4.108983791 +38410,Renagour Bridge,554836,56.17693404,-4.418601789 +37888,Rhidorroch,356990,57.90929699,-5.117024011 +36202,Robstone,133123,55.25880544,-4.807721027 +37484,Rosehall,234300,57.98210062,-4.586149187 +36362,Rowanburnfoot,133176,55.07463868,-2.918221754 +36366,Rusko,133177,54.90579321,-4.197617244 +37488,Sgodachail,234305,57.89327664,-4.546976399 +37494,Shenachie,234306,57.37739614,-3.954996467 +37498,Sheriffmills,234307,57.64636934,-3.351933142 +36212,Shewalton,133125,55.59709275,-4.627788758 +37502,Shielfoot,234310,56.76454538,-5.820892452 +36770,Shiplaw,14975,55.72796917,-3.207291434 +36216,Sills of Clyde,133126,55.66312399,-3.70505823 +37506,Skeabost,234313,57.45437017,-6.311235369 +36220,Spiers Bridge,133128,55.80186787,-4.32218584 +36412,Spilmersford,14871,55.9100322,-2.871497598 +36854,Sprouston,15012,55.61103047,-2.395690658 +36608,St Michaels,14930,56.3903691,-2.908114188 +37830,Stobs Castle,345032,55.38179195,-2.780516606 +36612,Strathmiglo,14931,56.27758355,-3.253078016 +37520,Strathpeffer,234316,57.59297854,-4.523041423 +37524,Strathy Bridge,234319,58.55872143,-4.002604762 +37534,Tarroul,234324,58.47619579,-3.267060495 +36826,Teviothead,14999,55.34222734,-2.936481216 +37542,Tromie Bridge,234327,57.0717185,-3.998638147 +36224,Tulliford Mill,133130,55.64374172,-3.75902119 +36442,Turnhouse,14883,55.94528452,-3.345442654 +38218,Unthank Bridge,506153,55.24336243,-2.967499315 +35960,Upper Monachyle,116007,56.39271229,-4.463706898 +36228,Victoria Bridge,133131,56.53983375,-4.813560642 +37552,Wardend Bridge,234331,57.58239454,-3.608335401 +36370,Waterhead,133178,55.37439338,-4.304549757 +36232,Waterside,133133,55.77868352,-4.240656047 +37982,Waterside Perth,369554,56.89929851,-2.820760629 +35966,Weisdale Mill,116008,60.2593269,-1.288604022 +37946,Wellsies Burn,368491,56.52284051,-3.293836481 +36236,Wellwood,133135,55.5109582,-4.123670365 +36642,Wester Cardean,14944,56.60590455,-3.151232647 +38192,Wheeb Bridge,502896,55.09004091,-4.661771619 +36432,Whitburn,14881,55.87157334,-3.682936938 +37866,White Bridge,355980,57.33362742,-4.673089381 +36374,White Laggan,133180,55.07382387,-4.401198238 +37558,Woodend,234333,57.0501291,-2.603567118 +36382,Woodfoot,133182,55.29395422,-3.41903487 +36870,Abbey St Bathans,15018,55.85329633,-2.387448356 +37132,Aberlour,234150,57.48021805,-3.205975095 +36730,Aberuchill,14966,56.37078118,-4.019153819 +38350,Almondbank at Main Street,535347,56.41717637,-3.516105507 +37152,Arcan Spillway,234154,57.55332889,-4.550906092 +36720,Auchinner,14964,56.31678388,-4.112254665 +36678,Auchnafree,14952,56.47302976,-3.904678161 +37610,Avonbank,322556,55.66347112,-4.051617294 +36654,Balbrogie,14947,56.57225831,-3.240802881 +36634,Balmacneil,14938,56.63825013,-3.664198017 +36262,Barsolus,133142,54.8658531,-4.951330607 +36270,Braidlie,133144,55.26142915,-2.824951121 +36752,Bridge of Earn,14971,56.3497307,-3.402582026 +37188,Bridge of Oich,234172,57.09229844,-4.74425988 +36532,Callander,14911,56.2422863,-4.214924958 +36670,Camusvrachan,14950,56.60039168,-4.250199477 +38178,Cleuchhead,499810,56.94774412,-2.388756894 +37222,Conon Bridge,234185,57.5694305,-4.440534753 +37232,Craigard,234189,57.06557358,-4.822135914 +38302,Crieff,520119,56.36081905,-3.847275886 +36734,Dalginross,14967,56.37511953,-3.978605821 +36710,Dalnamein,14960,56.79925858,-4.053453634 +36070,Dryfield,133080,55.93891825,-4.182216743 +38010,Erchless,377135,57.42804453,-4.623845183 +36658,Forter,14948,56.76674681,-3.332205729 +36814,Fountainhall,14991,55.73663595,-2.905930746 +38170,Foyers,498342,57.26265002,-4.483020773 +38436,Gairlochy,559194,56.91442441,-4.991445495 +37930,Garthdee,368014,57.11882365,-2.124396574 +38476,Glassgreen,581106,57.63198387,-3.296482414 +36862,Grantshouse,15015,55.88238352,-2.292035856 +36326,Greensands,133159,55.06942626,-3.617944341 +38428,Hottsbridge,559081,55.09041993,-3.167804617 +36786,Innerleithen,14981,55.61992749,-3.061056074 +37336,Inveran,234240,57.94318903,-4.410383325 +38354,Inverlaidnan Bridge,535718,57.26705143,-3.883273172 +37862,Inveroykel Bailey Bridge,354624,57.97362007,-4.604090887 +37696,Inzion,335641,56.68930326,-3.177597556 +37620,Irvine Waterside,327231,55.61682305,-4.674740715 +37822,Ivy Bridge,341495,57.31837309,-4.497591991 +38432,Johnstonehall,559082,55.06482387,-3.221128314 +37380,Kincraig,234254,57.12680642,-3.92521234 +36000,Kirkfieldbank,124855,55.67631471,-3.803247871 +35952,Kirkton Burn,116005,56.36683432,-4.377397065 +38038,Kirktown of Fetteresso,463690,56.96107386,-2.24381097 +36818,Lauder (New Mills),14996,55.73932168,-2.755976423 +37392,Levishie,234262,57.22003826,-4.647834071 +37954,Links Farm,368493,56.53282434,-3.339459645 +37884,Loch A Bhraoin,356328,57.72829261,-5.098319842 +38414,Loch Ard,554837,56.18379617,-4.422949664 +37406,Loch Flemington,234267,57.54236344,-3.993033026 +38086,Loch Maree,473675,57.6695583,-5.447154185 +37410,Loch Naver,234268,58.28571328,-4.389718176 +37850,Loch Tormasad,354622,57.56195723,-7.316740973 +38468,Loch Vaa,579464,57.23627646,-3.803196316 +477389,Loch Venachar,14898,56.23425763,-4.25686637 +35684,Lochindorb Level,115371,57.39798722,-3.714675739 +38240,Lochwinnoch,509677,55.78853152,-4.618943233 +36774,March Street,14976,55.6553291,-3.19057537 +37880,Meikle Kinord,356327,57.07826247,-2.921345756 +38206,Meikleholm Bridge,504722,55.36209214,-3.462380402 +36664,Moar,14949,56.57276553,-4.388958153 +36528,Nungate,14909,55.95402395,-2.771735305 +35500,Perth,10048,56.41191473,-3.434206313 +37746,Peter s Bridge,336365,57.59839179,-4.428885478 +37476,Red Bridge,234297,57.45461034,-4.494575811 +36206,Ross Priory,133124,56.05257642,-4.55746111 +38472,Royal Brackla Lade,581105,57.55531817,-3.887903021 +37514,Spey Dam,234315,57.01121062,-4.336367491 +38332,Sterling Mills Gauging Station,532968,56.14812902,-3.738809826 +37978,Stonyford,369551,56.84214794,-2.811870347 +37530,Struy,234321,57.42569662,-4.682400374 +36560,Tannadice,14918,56.7088965,-2.857771833 +36378,Whitesands,133181,55.06536955,-3.610749368 diff --git a/rivretrieve/uk_sepa.py b/rivretrieve/uk_sepa.py new file mode 100644 index 0000000..89b575c --- /dev/null +++ b/rivretrieve/uk_sepa.py @@ -0,0 +1,524 @@ +"""Fetcher for UK river gauge data.""" + +import logging +import re +from datetime import datetime, timedelta, timezone +from typing import List, Optional + +import pandas as pd +import requests + +from . import base, constants, utils + +logger = logging.getLogger(__name__) + + +class UKSEPAFetcher(base.RiverDataFetcher): + """Fetches river gauge data from the Scottish Environment Protection Agency (SEPA). + + Data Source: SEPA API (https://timeseriesdoc.sepa.org.uk/) + + Supported Variables: + - ``constants.DISCHARGE_DAILY_MEAN`` (m³/s) + - ``constants.DISCHARGE_INSTANT`` (m³/s) + - ``constants.STAGE_DAILY_MEAN`` (m) + - ``constants.STAGE_INSTANT`` (m) + """ + + API_LIMIT = 300000 + + METADATA_TRANSLATION_MAPPING = { + "station_id": constants.GAUGE_ID, + # "stationReference": "stationReference", + "station_name": constants.STATION_NAME, + "station_latitude": constants.LATITUDE, + "station_longitude": constants.LONGITUDE, + # "riverName": constants.RIVER, + # "catchmentArea": constants.AREA, + } + + def __init__(self, api_key: Optional[str] = None): + super().__init__() + self.client = _SEPAClient(api_key) + + @staticmethod + def get_cached_metadata() -> pd.DataFrame: + """Retrieves a DataFrame of available UK Environment Agency gauge IDs and metadata. + + This method loads the metadata from a cached CSV file located in + the ``rivretrieve/cached_site_data/`` directory. + + Returns: + pd.DataFrame: A DataFrame indexed by gauge_id, containing site metadata. + """ + return utils.load_cached_metadata_csv("uk_sepa") + + @staticmethod + def get_available_variables() -> tuple[str, ...]: + return ( + constants.DISCHARGE_DAILY_MEAN, + constants.STAGE_DAILY_MEAN, + constants.DISCHARGE_INSTANT, + constants.STAGE_INSTANT, + ) + + def get_metadata(self) -> pd.DataFrame: + """Fetches site metadata for all stations measuring level and/or flow from the SEPA API. + + Returns: + A pandas DataFrame indexed by gauge_id, containing site metadata. + """ + # These groups correspond to StationsWithFlow and StationsWithLevel + group_ids = ["270322", "615437"] + df_list = [] + for group_id in group_ids: + group_df = self.client._sepa_station_list(group_id=group_id) + df_list.append(group_df) + df = pd.concat(df_list) + df = df.drop_duplicates().reset_index(drop=True) + if df.empty: + return pd.DataFrame().set_index(constants.GAUGE_ID) + + df = df.rename(columns=self.METADATA_TRANSLATION_MAPPING) + return df.set_index(constants.GAUGE_ID) + + def _get_sepa_ts_name(self, variable): + if constants.INSTANTANEOUS in variable: + return "15minute" + elif f"{constants.DAILY}_{constants._MEAN}" in variable: + return "Day.Mean" + else: + raise ValueError(f"Unsupported variable: {variable}") + + def _get_sepa_parameter_name(self, variable): + if constants.STAGE in variable: + return "Level" + elif constants.DISCHARGE in variable: + return "Flow" + else: + raise ValueError(f"Unsupported variable: {variable}") + + def _download_data(self, gauge_id: str, variable: str, start_date: str, end_date: str) -> pd.DataFrame: + """Downloads the raw data from the SEPA API.""" + + if variable not in self.get_available_variables(): + raise ValueError(f"Unsupported variable: {variable}") + + ts_name = self._get_sepa_ts_name(variable) + stationparameter_name = self._get_sepa_parameter_name(variable) + + ts_list = self.client._sepa_timeseries_list( + station_id=gauge_id, + stationparameter_name=stationparameter_name, + ts_name=ts_name, + ) + if ts_list.shape[0] == 0: + return None + ts_id = ts_list["ts_id"].iloc[0] + ts_start = ts_list["from"].iloc[0] + ts_end = ts_list["to"].iloc[0] + + start_date_str = utils.format_start_date(start_date) + end_date_str = utils.format_end_date(end_date) + + start_date_dt = pd.to_datetime(start_date_str, utc=True) + end_date_dt = pd.to_datetime(end_date_str, utc=True) + + start_date_dt = max(ts_start, start_date_dt) + end_date_dt = min(ts_end, end_date_dt) + + # The request will fail if we ask for too much data, so we need to chunk our calls + if ts_name == "15minute": + delta = pd.Timedelta("15min") + elif ts_name.startswith("Day"): + delta = pd.Timedelta("1D") + + # Inclusive point count + n_points = ((end_date_dt - start_date_dt) // delta) + 1 + max_points = int(self.API_LIMIT * 0.9) + if n_points <= max_points: + chunks = [[start_date_dt, end_date_dt]] + else: + chunk_duration = (max_points - 1) * delta + chunks = [] + chunk_start = start_date_dt + while chunk_start <= end_date_dt: + chunk_end = min(chunk_start + chunk_duration, end_date_dt) + chunks.append([chunk_start, chunk_end]) + chunk_start = chunk_end + delta + + frames = [] + for chunk in chunks: + chunk_start = chunk[0].strftime("%Y-%m-%d") + chunk_end = chunk[1].strftime("%Y-%m-%d") + ts = self.client._sepa_timeseries_values( + ts_id=ts_id, start_date=chunk_start, end_date=chunk_end, metadata=True + ) + ts_parsed = self._parse_data(gauge_id=gauge_id, raw_data=ts, variable=variable) + frames.append(ts_parsed) + + out = pd.concat(frames) + out = out.sort_index() + out = out[~out.index.duplicated(keep="first")] + return out + + def _parse_data(self, gauge_id: str, raw_data: any, variable: str) -> pd.DataFrame: + """Parses the raw data into a standardized pandas DataFrame.""" + + df = raw_data[["Timestamp", "Value"]].copy() + + if constants.DAILY in variable: + df[constants.TIME_INDEX] = pd.to_datetime(df["Timestamp"]).dt.date + else: + df[constants.TIME_INDEX] = pd.to_datetime(df["Timestamp"]) + + df[variable] = pd.to_numeric(df["Value"], errors="coerce") + df = df[[constants.TIME_INDEX, variable]] + return df.set_index(constants.TIME_INDEX) + + def get_data( + self, + gauge_id: str, + variable: str, + start_date: Optional[str] = None, + end_date: Optional[str] = None, + ) -> pd.DataFrame: + """Fetches and parses time series data for a specific gauge and variable. + + This method retrieves the requested data from the provider's API or data source, + parses it, and returns it in a standardized pandas DataFrame format. + + Args: + gauge_id: The site-specific identifier for the gauge. + variable: The variable to fetch. Must be one of the strings listed + in the fetcher's ``get_available_variables()`` output. + These are typically defined in ``rivretrieve.constants``. + start_date: Optional start date for the data retrieval in 'YYYY-MM-DD' format. + If None, data is fetched from the earliest available date. + end_date: Optional end date for the data retrieval in 'YYYY-MM-DD' format. + If None, data is fetched up to the latest available date. + + Returns: + pd.DataFrame: A pandas DataFrame indexed by datetime objects (``constants.TIME_INDEX``) + with a single column named after the requested ``variable``. The DataFrame + will be empty if no data is found for the given parameters. + + Raises: + ValueError: If the requested ``variable`` is not supported by this fetcher. + requests.exceptions.RequestException: If a network error occurs during data download. + Exception: For other unexpected errors during data fetching or parsing. + """ + return self._download_data(gauge_id=gauge_id, variable=variable, start_date=start_date, end_date=end_date) + + +class _SEPAClient: + BASE_URL = "https://timeseries.sepa.org.uk/KiWIS/KiWIS" + + TOKEN_URL = "https://timeseries.sepa.org.uk/KiWebPortal/rest/auth/oidcServer/token" + + GARBAGE_STATION_PATTERNS = [r"^#", r"^--", r"testing", r"^Template\s", r"\sTEST$", r"\sTEMP$", r"\stest\s"] + + def __init__(self, api_key: Optional[str] = None): + self._auth_header = self._sepa_auth_headers(api_key) + + def _sepa_auth_headers(self, api_key: str | None = None) -> dict: + """Get access token to access the SEPA Timeseries API as a + https://timeseriesdoc.sepa.org.uk/api-documentation/before-you-start/what-controls-there-are-on-access/ + for more details. + """ + if api_key is None: + return {} + auth_headers = {"Authorization": "Basic " + api_key} + response = requests.post(self.TOKEN_URL, headers=auth_headers, data="grant_type=client_credentials") + response.raise_for_status() + access_token = response.json()["access_token"] + return {"Authorization": "Bearer " + access_token} + + def _sepa_group_list(self, timeout: int = 15) -> pd.DataFrame: + """Retrieve the list of available SEPA station groups from the KiWIS API.""" + params = { + "service": "kisters", + "datasource": 0, + "type": "queryServices", + "request": "getGroupList", + "format": "json", + "kvp": "true", + } + response = utils.requests_retry_session().get( + self.BASE_URL, params=params, headers=self._auth_header, timeout=timeout + ) + response.raise_for_status() + json_content = response.json() + column_names = list(map(str, json_content[0])) + rows = [dict(zip(column_names, row)) for row in json_content[1:]] + df = pd.DataFrame(rows) + return df + + def _sepa_station_list( + self, group_id: int | str | None = None, return_fields: str | List[str] | None = None, timeout: int = 15 + ) -> pd.DataFrame: + """Retrieve a list of SEPA hydrometric stations via the KiWIS API.""" + if return_fields is None: + return_fields = ["station_name", "station_no", "station_id", "station_latitude", "station_longitude"] + elif isinstance(return_fields, str): + return_fields = [f.strip() for f in return_fields.split(",")] + elif not isinstance(return_fields, (list, tuple)): + raise TypeError("return_fields must be a comma-separated string or a list/tuple of strings") + params = { + "service": "kisters", + "datasource": 0, + "type": "queryServices", + "request": "getStationList", + "format": "json", + "kvp": "true", + "returnfields": ",".join(return_fields), + } + + if group_id is not None: + params["stationgroup_id"] = group_id + + # Perform request + response = utils.requests_retry_session().get( + self.BASE_URL, params=params, headers=self._auth_header, timeout=timeout + ) + response.raise_for_status() + json_content = response.json() + + if not json_content or len(json_content) < 2: + return pd.DataFrame() + + # First row = column names + column_names = list(map(str, json_content[0])) + rows = [dict(zip(column_names, row)) for row in json_content[1:]] + df = pd.DataFrame(rows) + + # Convert lat/lon if present + for col in ["station_latitude", "station_longitude"]: + if col in df.columns: + df[col] = pd.to_numeric(df[col], errors="coerce") + + # Remove bogus or test stations + if "station_name" in df.columns: + pattern = re.compile("|".join(self.GARBAGE_STATION_PATTERNS), flags=re.IGNORECASE) + df = df[~df["station_name"].str.contains(pattern, na=False)] + + return df + + def _sepa_timeseries_list( + self, + station_id: str | None = None, + stationparameter_name: str | None = None, + ts_name: str | None = None, + coverage: bool = True, + group_id: str | None = None, + return_fields: str | List[str] | None = None, + timeout=15, + ): + """Query SEPA for available timeseries metadata.""" + # Check for missing input + if not any([station_id, ts_name, group_id]): + raise ValueError("No station_id, ts_name, or group_id provided.") + + # Handle user-provided return fields + if return_fields is None: + return_fields = [ + "station_name", + "station_id", + "stationparameter_name", + "ts_id", + "ts_name", + ] + elif not isinstance(return_fields, (str, list)): + raise TypeError("User-supplied `return_fields` must be a comma-separated string or list of strings.") + + # Convert list to comma-separated string + if isinstance(return_fields, list): + return_fields = ",".join(return_fields) + + # Remove any explicit 'coverage' mentions from return_fields + return_fields = return_fields.replace(",coverage", "").replace("coverage,", "").replace("coverage", "") + + # Base query + api_query = { + "service": "kisters", + "datasource": 0, + "type": "queryServices", + "request": "getTimeseriesList", + "format": "json", + "kvp": "true", + "returnfields": return_fields, + } + + # Handle inputs + if station_id is not None: + if isinstance(station_id, list): + station_id = ",".join(station_id) + api_query["station_id"] = station_id + + if stationparameter_name is not None: + api_query["stationparameter_name"] = stationparameter_name + + if ts_name is not None: + api_query["ts_name"] = ts_name + + if group_id is not None: + if station_id is not None or ts_name is not None: + raise ValueError("`group_id` cannot be used with `station_id` or `ts_name`.") + api_query["stationgroup_id"] = group_id + + if coverage: + api_query["returnfields"] = f"{api_query['returnfields']},coverage" + + # Perform request + response = utils.requests_retry_session().get( + self.BASE_URL, params=api_query, headers=self._auth_header, timeout=timeout + ) + response.raise_for_status() + json_content = response.json() + + if not isinstance(json_content, list) or len(json_content) < 2: + return pd.DataFrame() + + # Extract field names and rows + col_names = [str(c) for c in json_content[0]] + rows = [dict(zip(col_names, r)) for r in json_content[1:]] + df = pd.DataFrame(rows) + + # Convert numeric lat/lon + for col in ["station_latitude", "station_longitude"]: + if col in df.columns: + df[col] = pd.to_numeric(df[col], errors="coerce") + + # Convert coverage columns to datetime + for col in ["from", "to"]: + if col in df.columns: + df[col] = pd.to_datetime(df[col], errors="coerce", utc=True) + + return df + + def _sepa_timeseries_values( + self, + ts_id: str, + start_date: str | None = None, + end_date: str | None = None, + return_fields: str | List[str] | None = None, + metadata: bool = False, + md_return_fields: str | List[str] | None = None, + ca_sta: bool = False, + ca_sta_return_fields: str | List[str] | None = None, + timeout: int = 60, + ): + """Retrieve time series values from the SEPA KiWIS API.""" + # Default to past 24 hours + if start_date is None or end_date is None: + print("No start or end date provided, attempting to retrieve data for past 24 hours.") + end_date = datetime.now(tz=timezone.utc).date() + start_date = end_date - timedelta(days=1) + else: + start_date = pd.to_datetime(start_date).date() + end_date = pd.to_datetime(end_date).date() + + start_date_str = datetime.strftime(start_date, "%Y-%m-%d") + end_date_str = datetime.strftime(end_date, "%Y-%m-%d") + if ts_id is None: + raise ValueError("Please enter a valid ts_id.") + if isinstance(ts_id, list): + ts_id_str = ",".join(map(str, ts_id)) + else: + ts_id_str = str(ts_id) + + if return_fields is None: + return_fields = ["Timestamp", "Value"] + elif isinstance(return_fields, str): + return_fields = ["Timestamp", "Value"] + [return_fields] + elif isinstance(return_fields, list): + return_fields = ["Timestamp", "Value"] + return_fields + else: + raise TypeError("return_fields must be a string or list of strings.") + return_fields_str = ",".join(return_fields) + + # Handle metadata fields + if md_return_fields is None: + if metadata: + md_return_fields = [ + "ts_unitname", + "ts_unitsymbol", + "ts_name", + "ts_id", + "stationparameter_name", + "station_name", + "station_id", + ] + else: + md_return_fields = [] + md_return_fields = [f for f in md_return_fields if f != "ca_sta"] + + # Handle custom attributes + if ca_sta: + metadata = True + md_return_fields.append("ca_sta") + md_return_fields_str = ",".join(md_return_fields) + + if ca_sta_return_fields is None: + ca_sta_return_fields = ["CATCHMENT_SIZE", "GAUGE_DATUM"] + ca_sta_return_fields_str = ",".join(ca_sta_return_fields) + + # Build query + api_query = { + "service": "kisters", + "datasource": 0, + "type": "queryServices", + "request": "getTimeseriesValues", + "format": "json", + "kvp": "true", + "ts_id": ts_id_str, + "from": start_date_str, + "to": end_date_str, + "metadata": str(metadata).lower(), + "md_returnfields": md_return_fields_str, + "ca_returnfields": ca_sta_return_fields_str, + "returnfields": return_fields_str, + } + + # Perform request + r = utils.requests_retry_session().get( + self.BASE_URL, params=api_query, headers=self._auth_header, timeout=timeout + ) + r.raise_for_status() + json_content = r.json()[0] + + # Handle possible API errors + if len(json_content.keys()) == 3 and "message" in json_content: + raise RuntimeError(json_content["message"]) + + if "rows" in json_content: + num_rows = sum(map(int, json_content["rows"])) + if num_rows == 0: + raise RuntimeError("No data available for selected ts_id(s).") + + # Parse data + ts_cols = json_content["columns"] + if isinstance(ts_cols, list): # Actually not sure whether this would ever be the case + ts_cols = ts_cols[0] + ts_cols = ts_cols.split(",") + data = [dict(zip(ts_cols, row)) for row in json_content["data"]] + df = pd.DataFrame(data) + + # Convert to proper types + if "Timestamp" in df.columns: + df["Timestamp"] = pd.to_datetime(df["Timestamp"], errors="coerce", utc=True) + if "Value" in df.columns: + df["Value"] = pd.to_numeric(df["Value"], errors="coerce") + + # Add metadata + if metadata: + for field in md_return_fields: + if field != "ca_sta" and field in json_content: + df[field] = json_content[field] + + if ca_sta: + for field in ca_sta_return_fields: + if field in json_content: + df[field] = json_content[field] + + return df