Jonathan Coney

MRes Student in Climate and Atmospheric Science 2019-20

Index | About | Computer Project | Files

Computer Project Introduction | Get IDs | Split up IDs | Get Netatmo weather data

Get UK station IDs

This code calls the Netatmo API (you will need to specify your username, password, client_id, and client_secret from your Netatmo Developer Account on lines 47-50). It requires a .csv file containing known station MAC addresses, the path of which is specified in line 134. An example file can be found here. Due to the unique way the getpublicdata command works, some of the stations returned will be outside of the area specified, so some stations will be returned that are outside the UK.

This code requires the following core module:

and the following packages (I have included the versions that I am using):

Example input/output file

stations_modules.csv

Script

get_ids_modules.py 

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
#!/usr/bin/env python
# -*- coding=utf-8 -*-

#Code to add Netatmo stations to a csv file containing MAC addresses

##############################################################
###USAGE                                                   ###
###python get_ids_modules.py input_csv_file output_csv_file###
##############################################################


import sys
import requests
import pandas

def get_region(lat_ne, lon_ne, lat_sw, lon_sw):
    """
    get mac addresses of station and outdoor temperature sensor using getpublicdata
    in given box. Some stations returned outside of this.
    username, password, client id and client secret have been redacted. You can
    generate these by creating a Netatmo developer account at https://dev.netatmo.com/.

    Parameters
    ----------
    lat_ne : str(float or int)
        North eastern corner of rectangle. Latitude in degrees.
    lon_ne : str(float or int)
        North eastern corner of rectangle. Longitude in degrees.
    lat_sw : str(float or int)
        South western corner of rectangle. Latitude in degrees.
    lon_sw : str(float or int)
        South western corner of rectangle. Longitude in degrees.

    Raises
    ------
    NameError
        Handled by code in case a search yields no stations

    Returns
    -------
    dict containing station MAC addresses.

    """

    #setting up contact with API
    payload = {'grant_type': 'password',
                'username': "$user@example.com",
                'password': "$password",
                'client_id': "$client_id",
                'client_secret': "$client_secret",
                'scope': 'read_station'}

    try:
        #try to get access tokens
        response = requests.post("https://api.netatmo.com/oauth2/token",
                                 data=payload)
        response.raise_for_status()
        access_token = response.json()["access_token"]
        refresh_token = response.json()["refresh_token"]
        scope = response.json()["scope"]
    except requests.exceptions.HTTPError as error:
        print(error.response.status_code, error.response.text)

    params = {
        'access_token': access_token,
        'lat_ne' : lat_ne,
        'lon_ne' : lon_ne,
        'lat_sw' : lat_sw,
        'lon_sw' : lon_sw,
    }

    ids = {}
    NoResponse = True
    retry_count = 0
    while NoResponse:
        #try to get stations in given region, 5 attempts before moving on to next area
        try:
            #try to get stations
            response = requests.post("https://api.netatmo.com/api/getpublicdata", params=params)
            response.raise_for_status()
            data = response.json()["body"]
            for station in data:
            #find each value for each station
                _id = station['_id']
                mod = [n for n in station['modules'] if n.startswith('02:')]
                location = station['place']['location']
                altitude = station['place']['altitude']
                if 'city' in station['place'].keys():
                    city = station['place']['city']
                else:
                    city = 'no city'
                ids[_id] = ({'module_name':mod, 'location':location, 'altitude':altitude,
                             'city':city, 'full_modules':station['modules']})

            #Checking that some data has been returned
            if len(ids) == 0:
            #if everything works but we have no data returned in the given box, raise
                raise NameError('length')                
        except requests.exceptions.HTTPError as error:
            #if there's an error, try four more times before moving on
            print(error.response.status_code, error.response.text)
            if retry_count < 5:
                retry_count += 1
            else:
                return({})
        except NameError:
            if retry_count < 5:
                retry_count += 1
            else:
                return({})
        else:
            NoResponse = False
            return(ids)

def csvread(file_name):
    """
    import a previously saved csv file, in this case the file of stations

    Parameters
    ----------
    file_name : str
        Import a csv file containing Netatmo MAC addresses.

    Returns
    -------
    dict containing MAC addresses

    """
    file = pandas.read_csv(file_name, index_col=0).to_dict()
    return(file)
    
def get_stations():
    """
    Call getregion to ensure all of UK is covered without calling the API to 
    try and get data from the Atlantic Ocean.

    Returns
    -------
    dict containing MAC addresses.

    """
    master = {}
    areas = {49:[-7, -6, -5], 50: [-6, -5, -4, -3, -2, -1, 0, 1],
             51: [-6, -5, -4, -3, -2, -1, 0, 1, 2], 52:[-6, -5, -4, -3, -2, -1, 0, 1, 2],
             53: [-5, -4, -3, -2, -1, 0, 1], 54:[-8, -7, -6, -5, -4, -3, -2, -1, 0], 
             55:  [-8, -7, -6, -5, -4, -3, -2, -1], 56:[-7, -6, -5, -4, -3, -2], 
             57:[-9, -8, -7, -6, -5, -4, -3, -2, -1], 58: [-8, -7, -6, -5, -4, -3, -2], 
             59: [-4, -3, -2, -1], 60: [-3, -2, -1, 0]
             }
    for lat in areas:
        j = 0
        while j < len(areas[lat])-1:
            master.update(get_region(str(lat+1), str(areas[lat][j+1]), 
                                     str(lat), str(areas[lat][j])))
            j = j + 1
    return(master)
    
def mergedata(old_file, new_file):
    """
    Combining current station list ('old') with new fresh data from the API 
    ('new'), returning updated list ('merge') and saving csv file
  
    Parameters
    ----------
    old_file : str
        path to file containing Netatmo MAC addressed
    new_file : str
        path to file where output MAC addresses should be saved. Can be (and is
        in my case) the same as 'old_file'.

    Returns
    -------
    None.

    """
    old = csvread(old_file)
    new = get_stations()
    merge = old.copy()
    merge.update(new)
    df = pandas.DataFrame.from_dict(merge)
    df.to_csv(new_file)
    print(str(len(merge)-len(old))+' new stations added')
    print(str(len(merge))+' stations in total')
 
  
def main():
    inp=sys.argv[1]
    out=sys.argv[2]
    mergedata(inp,out)

if __name__ == "__main__":
    main()