Page History

To help users to improve S2S CMA MARS requests performance via the WebAPI.

A good understanding of the MARS efficiency issues is essential especially for users that are interested in downloading large amounts of data.

In general it is organised, as a huge tree, with the indentation below, showing different levels down that tree:

• centre (CMA, ECMWF, NCEP, JMA, ...)
  • realtime or reforecast
    •  type of data (control forecast or perturbed forecast)
      • type of level (single level or pressure level or potential  temperature)
        • model version date (2014-05-01 or ...)
          
          • hindcast dates (2014-01-01 or 2014-01-02 or 2014-01-03, ...)
            •  time-steps
              • members (for perturbed forecast)
                • levels (for pl or pt)
                  • parameters

The idea is to request as much data as possible from the same tape file. The natural way to group requests would be:
all parameters, all levels, all members, all time-steps for 1 hindcast date for a type of level for a type

Note the following:

1. 'all' means 'all' that the user wants. It doesn't have to be all parameters.
2. If a user is interested only on z500,  he may request more hindcast dates in one go, since the overall request will not be so big.

#!/usr/bin/env python
from ecmwfapi import ECMWFDataServer
modelVersionDate = "2014-05-01"  # This is the first model version for CMA (babj)
hindcastDate = "2014-04-01"      # The selected hindcast date
server = ECMWFDataServer()
server.retrieve({
    "class": "s2",
    "dataset": "s2s",
    "date": modelVersionDate,
    "expver": "prod",
    "hdate": hindcastDate, 
    "levtype": "sfc",
    "origin": "babj",
    "param": "165/166",
    "step": "24/to/1440/by/24",
    "stream": "enfh",
    "target": "data.pf.sfc",
    "time": "00",
    "number": "1/2/3",
    "type": "pf",
})

#!/usr/bin/env python
import calendar
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()

origin = "babj"
modelVersionDate = "2014-05-01"

def retrieve_data(hindcastDate,modelVersionDate,centrecma_reforecast():
    target = "target_s2s_%s_%s.grb""" % centre % hindcastDate
   
 server.retrieve({
      A "class": "s2",
       "dataset": "s2s",
  function to demonstrate how to iterate efficiently over all hindcastYears, hindcastMonths etc     "date": modelVersionDate,
       "expver": "prod",
  for a particular cma_reforecast_request.     "hdate": hindcastDate,
       "levtype": "sfc",
       "origin": centre,
  Change the variables below to adapt the iteration to your needs 
     "param": "165",
    hindcastYearStart = 1994
    "step": "0", hindcastYearEnd = 2014
    hindcastMonthStart = 1
  "stream": "enfh",
   hindcastMonthEnd = 12
    
    for hindcastYear "target": target,in list(range(hindcastYearStart, hindcastYearEnd + 1)):
       "time": "00", for hindcastMonth in list(range(hindcastMonthStart, hindcastMonthEnd + 1)):
       "type": "cf",
})

hindcastYearList = ["2012", "2013"]
hindcastMonthList = ["08", "09"]
hindcastDayList = ["01", "02"]

for hindcastYear in hindcastYearList     numberOfDays = calendar.monthrange(hindcastYear, hindcastMonth)[1]
            for hindcastDay in list(range(numberOfDays)):
   for     hindcastMonth in hindcastMonthList:
      hindcastDate for= hindcastDay'%04d%02d%02d' in% hindcastDayList:(
           hindcastDate =         hindcastYear, +hindcastMonth, hindcastMonthhindcastDay + hindcastDay1)
               retrieve_data(hindcastDate)

for hindcastYear in ["2012", "2013"]:
 target = "%s%s.grb" % (origin, hindcastDate)
               for hindcastMonth in ["08", "09"]:
 cma_reforecast_request(hindcastDate, target)

def cma_reforecast_request(hindcastDate, target):
    """       
        A CMA  if hindcastMonth in ["01", "03", "05", "07", "08", "10", "12"]: hindcastDateEnd = "31"
         if hindcastMonth in [ "04", "06", "09", "11"]: hindcastDateEnd = "30"reforecast request. Change the keywords below to adapt it to your needs.   
    """
    
    server.retrieve({
        "class": "s2",
        "dataset": "s2s",
        "date": modelVersionDate,
         for hindcastDay in ["01" ... hindcastDateEnd ]:"expver": "prod",
        "hdate": hindcastDate,
        "levtype": "sfc",
   hindcastDate = hindcastYear+hindcastMonth+hindcastDay
   "origin": origin,
        retrieve_data(hindcastDate,"2014-05-01","babj")

for hindcastYear in ["2012", "2013"]:"param": "165/166",
        "step": "24/to/1440/by/24",
    for  hindcastMonth in ["08stream",: "09enfh"]:,
         for hindcastDay in ["01"  "06" "11" "16" "21" "26"]:"target": target,
        "time": "00",
        "number": "1/2/3",
      hindcastDate = hindcastYear+hindcastMonth+hindcastDay  "type": "pf",
    })

if __name__ == '__main__':
     retrieve_data(hindcastDate,"2014-01-01","ammc"cma_reforecast()