Merge pull request #3987 from ONRR/dev

Dev-->Master

gatsby-config.js

@@ -10,11 +10,13 @@ const GOOGLE_ANALYTICS_ID = (process.env.google_analytics) ?
                 :
                 "UA-48605964-8";
 
+
+
 let config = {
   siteMetadata: {
     title: 'Natural Resources Revenue Data',
     description: 'This site provides open data about natural resource management on federal lands and waters in the United States, including oil, gas, coal, and other extractive industries.',
-    version: 'v5.1.1',
+    version: 'v5.1.2',
     googleAnalyticsId: GOOGLE_ANALYTICS_ID,
   },
   plugins: [
@@ -61,10 +63,17 @@ let config = {
     {
       resolve: `gatsby-source-filesystem`,
       options: {
-        name: `downloads`,
+        name: `downloads-revenues`,
         path: `${__dirname}/downloads/revenues`,
       },
     },
+    {
+      resolve: `gatsby-source-filesystem`,
+      options: {
+        name: `downloads-production`,
+        path: `${__dirname}/downloads/production`,
+      },
+    },
     'gatsby-transformer-excel',
     'gatsby-transformer-yaml',
     {
@@ -82,16 +91,7 @@ let config = {
         plugins: [],
       },
     },
-    {
-      resolve: 'custom-excel-transformer',
-      options: {
-        types: [
-          'ProductionVolumesXlsx__Sheet1',
-          'DisbursementsXlsx__Data',
-          'RevenueMonthlyXlsx__MontlyRev',
-          'FederalRevenueAcctYearFy061820181213Xlsx__FederalRevenuesFy06Fy18']
-      }
-    },
+    'custom-excel-transformer',
     `gatsby-plugin-meta-redirect`, // make sure to put last in the array
   ],
 };

plugins/custom-excel-transformer/data-transformers/constants.js

@@ -1,9 +1,9 @@
 /**
- * Constants that are only used by the data-transformers in this directory
+ * Constants that are only used by this plugin
  **/
 module.exports = Object.freeze({
   PRODUCTION_VOLUMES_EXCEL :'ProductionVolumesXlsx__Sheet1',
-  REVENUES_MONTHLY_EXCEL :'RevenueMonthlyXlsx__MontlyRev',
+  REVENUES_MONTHLY_EXCEL :'MonthlyRevenue032019Xlsx__MontlyRev',
   FEDERAL_DISBURSEMENTS_EXCEL:'DisbursementsXlsx__Data',
   FEDERAL_REVENUE_FY:'FederalRevenueAcctYearFy061820181213Xlsx__FederalRevenuesFy06Fy18'
 });

plugins/custom-excel-transformer/data-transformers/production-volumes-transformer.js

@@ -15,10 +15,12 @@ const CONSTANTS = require('../../../src/js/constants');
 const SOURCE_COLUMNS = {
 	Month: "Month",
 	CalendarYear: "Calendar Year",
+	FiscalYear: "Fiscal Year",
   ProductionDate: "Production Date",
   LandCategory: "Land Class",
   OnshoreOffshore: "Land Category",
   Commodity: "Commodity",
+  Product: "Product",
   Volume: "Volume",
 };
 
@@ -28,19 +30,31 @@ const SOURCE_COMMODITIES = {
   GasProductionVolume: "Gas Prod Vol (mcf)",
   CoalProductionVolume: "Coal Prod Vol (ton)",
 };
+/* List of all the products in the excel file and the corresponding column name */
+const SOURCE_COMMODITIES_FISCAL_YEAR = {
+  OilProductionVolume: "Oil (bbl)",
+  GasProductionVolume: "Gas (mcf)",
+  CoalProductionVolume: "Coal (tons)",
+};
 
 /* Map the source column name to the display name we want to use for that product */
 const SOURCE_COLUMN_TO_PRODUCT_DISPLAY_NAME = {
 	[SOURCE_COMMODITIES.OilProductionVolume]: "Oil",
 	[SOURCE_COMMODITIES.GasProductionVolume]: "Gas",
 	[SOURCE_COMMODITIES.CoalProductionVolume]: "Coal",
+	[SOURCE_COMMODITIES_FISCAL_YEAR.OilProductionVolume]: "Oil",
+	[SOURCE_COMMODITIES_FISCAL_YEAR.GasProductionVolume]: "Gas",
+	[SOURCE_COMMODITIES_FISCAL_YEAR.CoalProductionVolume]: "Coal",
 };
 
 /* Map the source column name to the units used for that product */
 const SOURCE_COLUMN_TO_PRODUCT_UNITS = {
 	[SOURCE_COMMODITIES.OilProductionVolume]: "bbl",
 	[SOURCE_COMMODITIES.GasProductionVolume]: "mcf",
 	[SOURCE_COMMODITIES.CoalProductionVolume]: "tons",
+	[SOURCE_COMMODITIES_FISCAL_YEAR.OilProductionVolume]: "bbl",
+	[SOURCE_COMMODITIES_FISCAL_YEAR.GasProductionVolume]: "mcf",
+	[SOURCE_COMMODITIES_FISCAL_YEAR.CoalProductionVolume]: "tons",
 };
 
 const PRODUCT_UNITS_TO_LONG_UNITS = {
@@ -67,28 +81,33 @@ const LOCATION_CATEGORY_TYPE_TO_PRODUCTION_CATEGORY ={
 }
 
 /* Use ES5 exports in order to be compatible with version 1.x of gatsby */
-module.exports = (node) => {
-	return createProductVolumeNodeByProduct(node);
+module.exports = (node, type) => {
+	return createProductVolumeNodeByProduct(node, type);
 }
 
-const createProductVolumeNodeByProduct = (productVolumeData) => {
-	if(productVolumeData[SOURCE_COLUMNS.Commodity] === undefined) return;
+const createProductVolumeNodeByProduct = (productVolumeData, type) => {
+	if(productVolumeData[SOURCE_COLUMNS.Commodity] === undefined && productVolumeData[SOURCE_COLUMNS.Product] === undefined) return;
 
   let node = {
 	  ProductionMonth: productVolumeData[SOURCE_COLUMNS.Month],
 	  ProductionYear: productVolumeData[SOURCE_COLUMNS.CalendarYear],
+	  FiscalYear: productVolumeData[SOURCE_COLUMNS.FiscalYear],
+	  LandClass: productVolumeData[SOURCE_COLUMNS.CalendarYear],
 	  LandCategory: productVolumeData[SOURCE_COLUMNS.LandCategory],
 	  OnshoreOffshore: productVolumeData[SOURCE_COLUMNS.OnshoreOffshore],
-	  ProductName: SOURCE_COLUMN_TO_PRODUCT_DISPLAY_NAME[productVolumeData[SOURCE_COLUMNS.Commodity]],
+	  ProductName: SOURCE_COLUMN_TO_PRODUCT_DISPLAY_NAME[productVolumeData[SOURCE_COLUMNS.Commodity]] || SOURCE_COLUMN_TO_PRODUCT_DISPLAY_NAME[productVolumeData[SOURCE_COLUMNS.Product]],
 	  Volume: productVolumeData[SOURCE_COLUMNS.Volume],
 	  internal: {
-	    type: `ProductVolumes`,
+	    type: type,
 	  }
   }
 
-	node.ProductionDate = new Date(node.ProductionYear, getMonthFromString(node.ProductionMonth));
+	node.ProductionDate = (node.ProductionYear) ? 
+		new Date(node.ProductionYear, getMonthFromString(node.ProductionMonth))
+		:
+		new Date(node.FiscalYear, 0);
 
-	node.Units = SOURCE_COLUMN_TO_PRODUCT_UNITS[productVolumeData[SOURCE_COLUMNS.Commodity]];
+	node.Units = SOURCE_COLUMN_TO_PRODUCT_UNITS[productVolumeData[SOURCE_COLUMNS.Commodity]] || SOURCE_COLUMN_TO_PRODUCT_UNITS[productVolumeData[SOURCE_COLUMNS.Product]];
 	node.LongUnits = PRODUCT_UNITS_TO_LONG_UNITS[node.Units];
 	node.LandCategory_OnshoreOffshore = 
 		node.LandCategory+( (node.OnshoreOffshore && node.LandCategory !== "Native American" )? " "+node.OnshoreOffshore.toLowerCase() : "" ) ;

plugins/custom-excel-transformer/data-transformers/revenues-transformer.js

@@ -67,6 +67,8 @@ const createRevenueNode = (revenueData, type) => {
 	  State: data[SOURCE_COLUMNS.State],
 	  County: data[SOURCE_COLUMNS.County],
 	  FiscalYear: data[SOURCE_COLUMNS.FiscalYear],
+	  Units: '$',
+	  LongUnits: 'dollars',
 	  OffshoreRegion: (data[SOURCE_COLUMNS.OffshoreRegion] === "" || data[SOURCE_COLUMNS.OffshoreRegion] === undefined) ?
 	  	data[SOURCE_COLUMNS.OffshoreRegion] : "Offshore "+data[SOURCE_COLUMNS.OffshoreRegion],
 	  internal: {
@@ -78,8 +80,31 @@ const createRevenueNode = (revenueData, type) => {
   let month = (revenueNode.Month)? getMonthFromString(revenueNode.Month) : 0;
 
   revenueNode.RevenueDate = new Date(year, month);
+
+  if(revenueNode.LandClass === CONSTANTS.NATIVE_AMERICAN) {
+    revenueNode.LandCategory = CONSTANTS.ONSHORE;
+  }
 
   revenueNode.RevenueCategory = LAND_CLASS_CATEGORY_TO_REVENUE_CATEGORY[revenueNode.LandClass] && LAND_CLASS_CATEGORY_TO_REVENUE_CATEGORY[revenueNode.LandClass][revenueNode.LandCategory];
+
+
+
+
+  if(revenueNode.RevenueCategory === undefined) {
+  	if(revenueNode.LandClass === CONSTANTS.NATIVE_AMERICAN) {
+  		revenueNode.RevenueCategory = CONSTANTS.NATIVE_AMERICAN;
+  	}
+  	else {
+  		revenueNode.RevenueCategory = 'Not tied to a lease';  	
+  	}
+  }
+
+  if(revenueNode.FiscalYear === undefined) {
+  	revenueNode.FiscalYear = (revenueNode.RevenueDate.getMonth() > 9 ) ? 
+  		(revenueNode.RevenueDate.getYear()+1901).toString()
+  		:
+  		(revenueNode.RevenueDate.getYear()+1900).toString();
+  }
 
 	return revenueNode;
 }

plugins/custom-excel-transformer/gatsby-node.js

@@ -7,12 +7,32 @@
  * are easily filtered, sorted and/or grouped. They also contain display ready names to be used
  * on the site.
  *
- * @TODO Now that are data sources are more standardized we may only want to add custom fields vs new nodes.
+ * @TODO Now that are data sources are more standardized we may only want to add custom fields vs new nodes or a schema?.
  *
  **/
 
 // Custom Data Transformers
-const DATA_TRANSFORMER_CONSTANTS = require('./data-transformers/constants');
+/**
+ * Constants that are only used by this plugin
+ **/
+const DATA_TRANSFORMER_CONSTANTS = {
+  PRODUCTION_VOLUMES_EXCEL :'ProductionVolumesXlsx__Sheet1',
+  PRODUCTION_VOLUMES_FY:'FederalProductionFy2009201820190326Xlsx__FyFederalProductionVolumes',
+  REVENUES_MONTHLY_EXCEL :'MonthlyRevenue032019Xlsx__MontlyRev',
+  FEDERAL_DISBURSEMENTS_EXCEL:'DisbursementsXlsx__Data',
+  FEDERAL_REVENUE_FY:'FederalRevenueAcctYearFy061820181213Xlsx__FederalRevenuesFy06Fy18',
+  NATIVE_AMERICAN_REVNUE_FY: 'NativeAmericanRevenuesXlsx__NativeAmericanRevenues'
+}
+
+const TRANSFORMER_NODE_TYPES = [
+	DATA_TRANSFORMER_CONSTANTS.PRODUCTION_VOLUMES_EXCEL,
+	DATA_TRANSFORMER_CONSTANTS.PRODUCTION_VOLUMES_FY,
+	DATA_TRANSFORMER_CONSTANTS.FEDERAL_DISBURSEMENTS_EXCEL,
+	DATA_TRANSFORMER_CONSTANTS.REVENUES_MONTHLY_EXCEL,
+	DATA_TRANSFORMER_CONSTANTS.FEDERAL_REVENUE_FY,
+	DATA_TRANSFORMER_CONSTANTS.NATIVE_AMERICAN_REVNUE_FY
+];
+
 const productionVolumesTransformer = require('./data-transformers/production-volumes-transformer');
 const revenuesTransformer = require('./data-transformers/revenues-transformer');
 const federalDisbursementsTransformer = require('./data-transformers/federal-disbursements-transformer');
@@ -23,11 +43,14 @@ async function onCreateNode(
 ) {
 	const { createNode } = actions;
 
-	if(options.types.includes(node.internal.type)) {
+	if(TRANSFORMER_NODE_TYPES.includes(node.internal.type)) {
 		let newNode;
 		switch(node.internal.type) {
 			case DATA_TRANSFORMER_CONSTANTS.PRODUCTION_VOLUMES_EXCEL:
-				newNode = productionVolumesTransformer(node);
+				newNode = productionVolumesTransformer(node, 'ProductVolumes');
+				break;
+			case DATA_TRANSFORMER_CONSTANTS.PRODUCTION_VOLUMES_FY:
+				newNode = productionVolumesTransformer(node, 'ProductVolumesFiscalYear');
 				break;
 			case DATA_TRANSFORMER_CONSTANTS.FEDERAL_DISBURSEMENTS_EXCEL:
 				newNode = federalDisbursementsTransformer(node);
@@ -38,6 +61,9 @@ async function onCreateNode(
 			case DATA_TRANSFORMER_CONSTANTS.FEDERAL_REVENUE_FY:
 				newNode = revenuesTransformer(node, 'ResourceRevenuesFiscalYear');
 				break;
+			case DATA_TRANSFORMER_CONSTANTS.NATIVE_AMERICAN_REVNUE_FY:
+				newNode = revenuesTransformer(node, 'ResourceRevenuesFiscalYear');
+				break;
 		}
 
 		newNode.id = createNodeId(node.id);

src/components/locations/SectionDisbursements.js

@@ -30,7 +30,7 @@ const SectionDisbursements = props => {
     	if (usStateDisbursements && offshoreDisbursements > 0) {
       content = <div>
         <p>
-							ONRR also disburses some revenue from natural resource extraction to state governments. <strong>In { year }, ONRR disbursed {utils.formatToDollarInt(allDisbursements)} to {usStateData.title}.</strong>
+							ONRR also disburses some revenue from natural resource extraction to state governments. <strong>In { year }, ONRR disbursed {utils.formatToDollarInt(allDisbursements)} to {usStateData.title}. </strong>
 							This included revenues from both onshore and offshore extraction in or near {usStateData.title}:
         </p>
         <ul>