65C2F32D7BC372KWZGU2DRLAE7BPFOHBXDSN2FTEHWNGB2FO6MBQC R2OU7BE4I4QC5IRZU7BJGHJQNTBNF3265LOHZB3QQDQD7OL46G5AC GQVS55HIQLU7KPJNRMF57QUM4EATSWFQRCS7ZEJMJPUXFX2NHSYAC YAE43L6AF2HPQYJZDQGFYGFEV7722SUMAXXMP2ZQZMPE7VC6EWKAC XI5ALEH6NPTQWB6O62QV62EP4H3K7WSNTHCOGT3LZIIU6I2YDGIQC TTR5IFSG25VNBQ2F2FNOLUMTEIHVBHFOEXYB2ZWEHWOURUV4GJMQC 4QOTH75I2VINLY52J75OR3E3B5CN5ITHJGOAKQYXI6PH3XLJ6DBQC BB2T6X3XK3Y4JFOMBGKTFVL6BZ5NOGBTPSCK4ADRSZQLQKQHESLAC SPSFTMLRZE2R4EBAAQKWBUZDRYZ36S2VFTBIJRE6XS7JMKELV4AQC EHEK63WARL6PGYZMCSOH53JXOYPHNGWZO4QP5RKODPIQJ53IWDDQC 5B2HBV3JTNBNJYEJ4BXFQDEP2D552SBJFPY6STKSK7SPFZYSTCFAC PK7JY27R55PZON4SPFLJBX66IAE47SXBRAGDHXM4SN6TKZXO5LMAC XPXYFEZMZJSF2GOFG5HTCMLVHINNVRYIVWV7KJ6A7A3PG3CM2TGAC PQKGZNQGSRWMV3D6AUUBH3JUICKXXKDV4OLLDUM3YP5ESORBYKLAC E2T2A74YPYLDPT2FCBLRJSJTDND2D5EDDRSXJZCUOQVEEIVCM2MAC TSY4YBBZ4AEW2JTEI4AS5FGTTUZR5WE3TENDHVIOWWY6V2DRT7ZQC 2CCG6KUP6VL2Q7WQKLVNIPPGZWHSWIOWETRCS3APZTV4WSF5GLWAC 4MG5JFXTKAE3SOVKGGNKEUTNCKOWEBHTGKVZHJWLWE3PTZTQKHPAC GUXZCEWWPBCHXO26JVWZ74CTSDFDDO775YR7FKY7UGVZCA7GCSYAC SAHJYVNBUBBIUBI4ZMAXK4QJFOT54M5UA3W2HQMTNDSP3GGCRX7QC CUADTSHQNPGMWHIJCWXKHNNIY4UJOYZ7XBZA5CJ2VVJJAMSNT4BAC 2SABVMY3A2RZDF3KJXZSMJ2UQ4Q5EW422G4DVBJRKK26S2ESGVQAC 77SIQZ3EGGV6KSECMLPDKQFGEC7CCFAPWGER7ZARQ5STDKJNU6GQC AIFRDCG2GLASIMF3WZYAQBHIGZZDGPMP2WBYH7HXIYORKAK2SUYAC UCSU3QE4352YC4VOVI7GCSTNNA5FAHYURNJLJQEHY32BB6S3CZVAC 5POF332LJEBGWEJUGI34P33Q4BQP7CAQNV5ODQT3PWG7FI7VNLOQC UKQAGL5F5LWZZR7RWFJI3H2MW6LXPRGPXAIGNBZI5IVJLWUFHLGAC UUD3CJZLSTAVLMIXIWE4CHN5HQSFM6K3DCPWECJMKGXDOROK4G4AC K4CH53V4MO5KCCJOOQUQKI3LEFSUSCNAJS24VUWOZISXTCQD4FZQC use crate::{// DataSpec,DataType,// DateRange,// data_from_file,// MonthlyDate,SeriesId,SeriesJson,SeriesSpec,Transform,};
use crate::{ GraphicRange, SeriesId, SeriesSpec, Transform };
kt.to_ref().try_into().map_err(|err: keytree::error::Error| {keytree_error(file!(), line!(), &err.to_string())})
let mut spec: Spec = kt.to_ref().try_into().map_err(|err: keytree::error::Error| {external(file!(), line!(), &err.to_string())})?;// We need to do some post-processing to// Iterate over PageSpecs andspec.0.iter_mut().for_each(|page_spec| page_spec.downcast_countries());Ok(spec)
let text_spec = match graphic_spec.category_opt {Some(GraphicCategory::Cleaned) => TextSpec::Link,Some(GraphicCategory::Collation) => TextSpec::Link,Some(GraphicCategory::Source) => TextSpec::Meta,
let caption_spec = match graphic_spec.category_opt {Some(GraphicCategory::Cleaned) => CaptionSpec::Link,Some(GraphicCategory::Collation) => CaptionSpec::Link,Some(GraphicCategory::Source) => CaptionSpec::Meta,
pub fn new(country: Country,data_type: DataType,index: usize,height_opt: Option<f32>) -> Self{PageSpec {country,data_type,index,height_opt,seriess: BTreeMap::new(),graphics: Vec::new(),}}// Because country is specified in the PageSpec level of a specification, on initialization of a// PageSpec, we need to set the country in series. The only way to initialize is through the// Spec::from_file() function.pub (crate) fn downcast_countries(&mut self) {for mut country in self.seriess.iter().map(|(_, series_spec)| series_spec.country) {country = Some(self.country);}}
"link" => Ok(TextSpec::Link),"meta" => Ok(TextSpec::Meta),_ => Err(keytree_error(file!(), line!(), "Failed to parse to Text")),
"link" => Ok(CaptionSpec::Link),"meta" => Ok(CaptionSpec::Meta),_ => Err(external(file!(), line!(), "Failed to parse to Text")),
TextSpec::Link => kt.push_value(0, "text", "link"),TextSpec::Meta => kt.push_value(0, "text", "meta"),TextSpec::None => {},
CaptionSpec::Link => kt.push_value(0, "caption", "link"),CaptionSpec::Meta => kt.push_value(0, "caption", "meta"),CaptionSpec::None => {},
use std::convert::TryInto;use fred_api::Fred;use keytree::{ KeyTreeRef };use time_series::{ DatePoint, RegularTimeSeries, TimeSeries };use crate::SeriesSpec;use crate::error::*;// --- Specifications -----------------------------------------------------------------------------#[derive(Clone, Copy, Debug)]pub enum ParserSpec {FredSpec(FredSpec),FredDailySpec(FredDailySpec),}#[derive(Clone, Copy, Debug)]pub struct FredSpec {drop_first: usize}impl FredSpec {pub fn empty() -> Self {FredSpec { drop_first: 0 }}}#[derive(Clone, Copy, Debug)]pub struct FredDailySpec;impl FredDailySpec {pub fn empty() -> Self {FredDailySpec}}impl<'a> TryInto<FredSpec> for KeyTreeRef<'a> {type Error = keytree::Error;fn try_into(self) -> Result<FredSpec, Self::Error> {let drop_first: usize = self.opt_value("parser::drop_first")?.unwrap_or(0);Ok(FredSpec { drop_first })}}pub struct FredLineParser;impl FredLineParser {pub fn default() -> Self {FredLineParser}}pub enum FredLine {Ok(DatePoint::<1>),Dot(time_series::MonthlyDate),}impl FredLine {fn to_datepoint(&self) -> Option<DatePoint::<1>> {match self {FredLine::Ok(dp) => Some(*dp),FredLine::Dot(_) => None,}}fn new(obs: &fred_api::Observation, line: usize) -> Result<FredLine, Error> {let year = obs.date[..4].parse().map_err(|_| {parse_datepoint_failed(file!(),line!(),&line.to_string(),&format!("{}, {}", obs.date, obs.value),)})?;let month = obs.date[5..7].parse().map_err(|_| {parse_datepoint_failed(file!(),line!(),&line.to_string(),&format!("{}, {}", obs.date, obs.value),)})?;let date = time_series::MonthlyDate::ym(year, month);match obs.value.as_str() {"." => Ok(FredLine::Dot(date)),_ => {let value = obs.value[12..].parse::<f32>().map_err(|_| {parse_datepoint_failed(file!(),line!(),&line.to_string(),&format!("{}, {}", obs.date, obs.value),)})?;Ok(FredLine::Ok(DatePoint::new(date, [value])))},}}}// I'm not sure if its necessary to separate out FredParser from its specification, but it 'feels// right' at this point. Maybe fix in future?pub struct FredParser(pub FredSpec);impl FredParser {/// Given the series specification and the parser type, fetch the data and build it into a/// `RegularTimeSeries`.pub (crate) fn to_data(&self, spec: &SeriesSpec) -> Result<RegularTimeSeries::<1>, Error> {// First fetch the datalet observations = match Fred::series_observations(&spec.series_id.to_string()) {Ok(series_obs) => series_obs.observations,Err(err) => {return Err(fred_error(file!(),line!(),&err.to_string(),))},};// Iterate through observations to build time_series.let mut ts = TimeSeries::<1>::new(Vec::new());for (line_num, obs) in observations.iter().enumerate().skip(self.0.drop_first){let dp = FredLine::new(obs, line_num + 1)?.to_datepoint().ok_or_else(|| {expected_datapoint_found_dot(file!(),line!(),&format!("{}, {}", obs.date, obs.value),&(line_num + 1).to_string(),)})?;ts.push(dp);}// Convert time-series to regular time-series.ts.try_into().map_err(|err: time_series::error::Error| external(file!(), line!(), &err.to_string()))}// /// Parse observations into string for debugging and exit.// pub fn soft_parse(&self, observations: fred_api::Observations, err: Error) {// for (i, obs) in observations.iter().enumerate() {// eprintln!(// "{} {}, {}",// i + 1,// obs.date,// obs.value,// )// }// eprintln!("{}", err);// std::process::exit(1);// }}// --- Parsers ------------------------------------------------------------------------------------pub enum Parser {FredParser,FredDailyParser,}pub struct FredDailyParser(pub FredDailySpec);impl FredDailyParser {pub (crate) fn to_data(&self, series_spec: &SeriesSpec) -> Result<RegularTimeSeries::<1>, Error> {let observations = match Fred::series_observations(&series_spec.series_id.to_string()) {Ok(series_obs) => series_obs.observations,Err(err) => { return Err(fred_error(file!(), line!(), &err.to_string())) },};let mut ts = TimeSeries::<1>::new(Vec::new());let mut month_data = Vec::new();let mut current_date = time_series::MonthlyDate::ym(0,0);for (i, obs) in observations.iter().enumerate() {// Get the Datapoint or else continue.match FredLine::new(obs, i + 1)?.to_datepoint() {Some(dp) => {if dp.date() != current_date {let value = month_data.iter().sum::<f32>() / month_data.len() as f32;ts.push(DatePoint::<1>::new(current_date, [value]));current_date = dp.date();} else { month_data.push(dp.value(0)); }},None => {}, // Ignore dot line.}}ts.try_into().map_err(|err| {expected_regular_time_series(file!(),line!(),&series_spec.country().to_string(),&series_spec.data_type.to_string(),&series_spec.series_id.to_string(),)})}}
use fred_api::Fred;use keytree::{KeyTree,KeyTreeRef,};use keytree::serialize::{IntoKeyTree,KeyTreeString,};use time_series::{DatePoint,RegularTimeSeries,TimeSeries,};
use fred_api::{ Fred };use keytree::{ KeyTree, KeyTreeRef };use keytree::error::keytree_selection_failed;use keytree::serialize::{ IntoKeyTree, KeyTreeString };use time_series::{ RegularTimeSeries, TimeSeries };
use crate::ts::{GraphicCategory,PageSpec,Transform,};
use crate::parser::*;use crate::ts::{ GraphicCategory, Transform };
/// Return the pathname of a data file.pub fn data_path(root_path: &str,data_type: DataType,country: Country,series_id: SeriesId,extension: &str) -> String{format!("{}/{}/{}/{}.{}",root_path,data_type,country.as_path(),series_id,extension,)}
let year = s[..4].parse().map_err(|_| parse_date(file!(), line!(), "failed"))?;let month = s[5..7].parse().map_err(|_| parse_date(file!(), line!(), "failed"))?;
let year = s[..4].parse().map_err(|_| {parse_datepoint_failed(file!(),line!(),"?",s,)})?;let month = s[5..7].parse().map_err(|_| {parse_datepoint_failed(file!(),line!(),"?",s,)})?;
let mut page_spec = PageSpec {country: *country,data_type: *data_type,index: 0,height_opt: None,seriess: BTreeMap::new(),graphics: Vec::new(),};
let mut page_spec = ts::PageSpec::new(*country, // country*data_type, // data_type0, // indexNone, // height);
match self.get_series_spec(&series_id) {Some(_) => {},None => {match fs::remove_file(entry.path()) {Ok(_) => {},Err(_) => {return Err(file_error(file!(), line!()))},
match self.get_series_spec(&series_id) {Some(_) => {},None => {println!("remove file: {}", entry.path().display());// match fs::remove_file(entry.path()) {// Ok(_) => {},// Err(_) => {// return Err(file_error(file!(), line!()))// },// }},
// We conflate both FRED-facing series and client-facing series. The reason for this is so that// the time_series() functionality is in one place. The down-side is that we need to make// the transforms optional, as they are not used in FRED-facing functionality, and// the country field optional as it is specified at a higher level in JsonSpec.
// The country field is always `Some`. The `Option` is used in initialization from a keytree file.// The transforms field is set is the time-series and ui specifications, and so is empty when a// SeriesSpec is initialized from a source specification.
/// Read metadata from file.pub fn read_meta_from_file(&self, country: Country, root_path: &str) -> SeriesMetaData{let path = data_path(
pub fn country(&self) -> Country {self.country.unwrap()}pub fn data_path(&self, root_path: &str, extension: &str) -> String {format!("{}/{}/{}/{}.{}",
"meta");let meta_str = fs::read_to_string(path).unwrap();
extension,)}pub fn dir_path(&self, root_path: &str) -> String {format!("{}/{}/{}",root_path,self.data_type,self.country().as_path(),)}/// Read metadata from file.pub fn read_meta_from_file(&self, root_path: &str) -> SeriesMetaData{let meta_str = fs::read_to_string(&self.data_path(root_path, "meta")).unwrap();
let csv_path = data_path(root_path,self.data_type,// Expect country to be Some while building source.self.country.unwrap(),self.series_id.clone(),"csv",);let meta_path = data_path(root_path,self.data_type,// Expect country to be Some while building source.self.country.unwrap(),self.series_id.clone(),"meta",);
let series_id_stem = self.series_id.stem();let csv_path = data_path(root_path,self.data_type,country,series_id_stem,"csv",);let ts = TimeSeries::<1>::from_csv(&csv_path)
let ts = TimeSeries::<1>::from_csv(&self.data_path(root_path, "csv"))
// Select the observation data from the FRED data.let observations = match Fred::series_observations(&self.series_id.to_string()) {Ok(series_obs) => series_obs.observations,Err(err) => { return Err(fred_error(file!(), line!(), &err.to_string())) },};// We need to build a RegularTimeSeries here, and then use RegularTimeSeries here// to build csv file.// Data is parsed into a RegularTimeSeries// Want power to drop_firstlet mut v = Vec::new();// Drop the first n itemslet skip = match self.drop_first {None => 0,Some(n) => n,};for (i, obs) in observations.iter().enumerate().skip(skip) {let date = MonthlyDate::from_str(&obs.date)?;let value: f32 = match obs.value.parse() {Ok(n) => n,Err(_) => {let err = parse_fred_value_failed(file!(),line!(),&self.data_type.to_string(),// Expect country to be Some while building source.&country.to_string(),&self.series_id.to_string(),&format!("{}, {}",obs.date,obs.value,),i + 1,);self.soft_parse(observations, err);unreachable!();},};let date_point = DatePoint::<1>::new(date.0, [value]);v.push(date_point)
// We want to handle data fetching and parsing in the Parser implementation, so make the// Parser the receiver.match self.parser {ParserSpec::FredSpec(fred_spec) => FredParser(fred_spec).to_data(self),ParserSpec::FredDailySpec(fred_daily_spec) => FredDailyParser(fred_daily_spec).to_data(self),
let ts = TimeSeries::new(v);let rts = match ts.try_into() {Ok(rts) => {rts},Err(_) => {let err = expected_regular_time_series(file!(),line!(),&country.to_string(),&self.data_type.to_string(),&self.series_id.to_string(),);self.soft_parse(observations, err);unreachable!();},};Ok(rts)
// pub fn data_without_transform(// &self,// country: Country,// root_path: &str) -> Result<RegularTimeSeries<1>, Error>// {// // Select the observation data from the FRED data.//// let observations = match Fred::series_observations(&self.series_id.to_string()) {// Ok(series_obs) => series_obs.observations,// Err(err) => { return Err(fred_error(file!(), line!(), &err.to_string())) },// };
// // We need to build a RegularTimeSeries here, and then use RegularTimeSeries here// // to build csv file.// // Data is parsed into a RegularTimeSeries// // Want power to drop_first// let mut v = Vec::new();// // Drop the first n items// let skip = match self.drop_first {// None => 0,// Some(n) => n,// };// for (i, obs) in observations.iter().enumerate().skip(skip) {// let date = MonthlyDate::from_str(&obs.date)?;// let value: f32 = match obs.value.parse() {// Ok(n) => n,// Err(_) => {// let err = parse_fred_value_failed(// file!(),// line!(),// &self.data_type.to_string(),// // Expect country to be Some while building source.// &country.to_string(),// &self.series_id.to_string(),// &format!(// "{}, {}",// obs.date,// obs.value,// ),// i + 1,// );// self.soft_parse(observations, err);// unreachable!();// },// };// let date_point = DatePoint::<1>::new(date.0, [value]);// v.push(date_point)// }// let ts = TimeSeries::new(v);// let rts = match ts.try_into() {// Ok(rts) => {// rts// },// Err(_) => {// let err = expected_regular_time_series(// file!(),// line!(),// &country.to_string(),// &self.data_type.to_string(),// &self.series_id.to_string(),// );// self.soft_parse(observations, err);// unreachable!();// },// };// Ok(rts)// }/// Parse observations into string for debugging and exit.pub fn soft_parse(&self, observations: fred_api::Observations, err: Error) {for (i, obs) in observations.iter().enumerate() {eprintln!("{} {}, {}",i + 1,obs.date,obs.value,)}eprintln!("{}", err);std::process::exit(1);}
let mut csv = String::new();for date_point in rts.iter(DateRange::new(&None, &None).0) {let date = date_point.date();let date_str = format!("{}-{:02}-01",date.year(),date.month(),);
// Build csv manually
let line = format!("{}, {}\n",date_str,date_point.value(0),);csv.push_str(&line);
let mut s = String::new();for dp in rts.iter(time_series::DateRange::new(&None, &None)) {s.push_str(&format!("{}-{}-01, {}",dp.date().year(),dp.date().month(),dp.value(0).to_string()))
let dir_path = &format!("{}/{}/{}",root_path,self.data_type,self.country.unwrap().as_path(),);
fs::create_dir_all(&self.dir_path(root_path)).map_err(|err| create_dir_failed( file!(), line!(), &err.to_string()))?;
let filename = &format!("{}/{}.csv",dir_path,self.series_id,);if let Err(err) = fs::create_dir_all(&dir_path) {return Err(failed_to_create_dir(file!(),line!(),&err.to_string()))}
fs::write(self.data_path(root_path, "csv"), s).map_err(|err| external(file!(), line!(), &err.to_string()))?;
let meta = match Fred::series(&self.series_id.to_string()) {Ok(series) => {
let series = Fred::series(&self.series_id.to_string()).map_err(|err| external(file!(), line!(), &err.to_string()))?;let series_item = series.seriess.iter().next().ok_or_else(|| {expected_series_data(file!(), line!(), &self.series_id.to_string())})?;
let series_item = series.seriess.iter().next().unwrap();SeriesMetaData {realtime: series.realtime_start.clone(),series_id: self.series_id.clone(),title: series_item.title.clone(),observation_start: series_item.observation_start.clone(),observation_end: series_item.observation_end.clone(),frequency: series_item.frequency.clone(),seasonal_adjustment: series_item.seasonal_adjustment.clone(),}},Err(err) => { return Err(fred_error(file!(),line!(),&err.to_string()))},
let meta = SeriesMetaData {realtime: series.realtime_start.clone(),series_id: self.series_id.clone(),title: series_item.title.clone(),observation_start: series_item.observation_start.clone(),observation_end: series_item.observation_end.clone(),frequency: series_item.frequency.clone(),seasonal_adjustment: series_item.seasonal_adjustment.clone(),
let dir_path = &format!("{}/{}/{}",root_path,self.data_type,// Expect country to be Some while building source.self.country.unwrap().as_path(),);
fs::create_dir_all(self.dir_path(root_path)).map_err(|err| create_dir_failed( file!(), line!(), &err.to_string()))?;let path = self.data_path(root_path, "meta");
// let parser_str_opt: Option<String> = self.opt_value("series::parser")?;// let parser = match parser_str_opt {// None => Parser::Fred,// Some(parser_str) => {// Parser::from_str(&parser_str)// .map_err(|err| keytree::error::external(file!(), line!(), &err.to_string()))?// },// };let a_parser: Option<String> = self.opt_value("series::parse_cat::fred_daily")?;let b_parser: Option<String> = self.opt_value("series::parse_cat::fred")?;
let parser = match (a_parser.is_some(), b_parser.is_some()) {(false, false) => ParserSpec::FredDailySpec(FredDailySpec::empty()),(true, false) => ParserSpec::FredDailySpec(FredDailySpec::empty()),(false, true) => ParserSpec::FredSpec(self.at("series::parser")?),(true, true) => Err(keytree_selection_failed(file!(), line!()))?,};
}#[derive(Clone, Copy, Debug, Serialize)]/// Specifies the range of a graphicpub struct GraphicRange {min: f32,max: f32,}impl FromStr for GraphicRange {type Err = Error;fn from_str(s: &str) -> Result<Self, Self::Err> {let segment: Vec<&str> = s.split(" to ").collect();if segment[0].is_empty() || segment[1].is_empty() {return Err(parse_graphic_range_failed(file!(),line!(),s))};let min = segment[0].parse().map_err(|_| parse_graphic_range_failed(file!(), line!(), s))?;let max = segment[1].parse().map_err(|_| parse_graphic_range_failed(file!(), line!(), s))?;Ok(GraphicRange { min, max })}
impl fmt::Display for GraphicRange {fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {write!(f, "{} to {}", self.min, self.max)}}// /// The United States prime interest rate data is daily. To buid a monthly time-series, we read// /// through raw csv data, calculate a monthly value and add to to the time-series. The data include// /// missing days, so we need the mechanism to ignore datepoints with value ".".//// ts.try_into().map_err(|err: time_series::error::Error| {// external(// file!(),// line!(),// &err.to_string(),// )// })// }
tags: self.vec_value("series::tag")?,enumerate: self.vec_value("series::enumerate")?,exclude: self.vec_value("series::exclude")?,require: self.vec_value("series::require")?,
tags: self.opt_vec_value("series::tag")?,enumerate: self.opt_vec_value("series::enumerate")?,exclude: self.opt_vec_value("series::exclude")?,require: self.opt_vec_value("series::require")?,
))}pub fn failed_to_serialize_to_json(code_file: &str,code_line: u32) -> Error{Error(format!("[ui_data:09:{}:{}] Failed to serialize to json.",code_file,code_line,