Page History

Motivation:
- current way of managing job config via files makes it hard to manage job configurations, persisting it in MySQL would make it easier to manage and also provide various CRUD API interfaces to mange the job configuration.
Proposed Change:
- create MySQL table `gobblin_job_config`
- create REST endpoints for CRUD operations
- add functionality to detect change in any job config and schedule or trigger job accordingly ( similar to current mechanism based on file change )
- In future, Gobblin should use JDBI (v3) as ORM instead of plan JDBC based persistence which will become cumbersome as it uses DB heavily for all kinds of metadata storage.
New or Changed Public Interfaces:
- POST /job/{jobName}
- PUT /job//{jobName}
- GET /job/{jobName}
- DELETE /job/{jobName}
Migration Plan and Compatibility:
- since this is new functionality, in addition to the current file based job config, user can select either one to manage the job.
Rejected Alternatives:
- prefer use of other REST framework ( preferring to use REST.li ( instead of JDBI) as it is dropwizard or any other as its built in framework for other APIs

...

Code Block

language	sql
firstline	1
title	gobblin_job_config

CREATE TABLE `gobblin_job_store` (
  `job_id`_owners` (
     `name` varchar(255) NOT NULL DEFAULT '',
     `email` varchar(255) NOT NULL DEFAULT '',
     `team_name` int(11) DEFAULT NULL,
     `team_email` int(11) DEFAULT NULL,
     `org_name` int(11) DEFAULT NULL,
     `created_date` timestamp NOT NULL AUTO_INCREMENT,
  `job_name`DEFAULT CURRENT_TIMESTAMP,
     `updated_date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
     PRIMARY KEY (`email`)
);

CREATE TABLE `gobblin_sync_systems` (
    `name` varchar(255) NOT NULL DEFAULT '',
    `job`db_description`type` varchar(255) NOT NULL DEFAULT '',
    `users` varchar(255) NOT NULL DEFAULT '',
  `job_is_disabled`  `on_hold` tinyint(1) NOT NULL DEFAULT '0',
    `deprecated` tinyint(1) NOT NULL DEFAULT '0',
    `config` text,
    `created_date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
    `updated_date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
    PRIMARY KEY (`job_name`)`name`)
);

CREATE TABLE `gobblin_jobs` (
    `id` int(11) NOT NULL AUTO_INCREMENT,
  UNIQUE KEY `job_id_index``name` varchar(`job_id`255) USINGNOT BTREE
) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8;


CREATE TABLE `gobblin_job_properties_new
` (
  `job_name` varchar(255) NOTNULL,
    `description` varchar(255) DEFAULT NULL,
    `schedule` varchar(64) DEFAULT NULL,
    `is_disabled` tinyint(1) DEFAULT '0',
    `priority` smallint(6) DEFAULT NULL,
    `configs` text,
    `owner_email` int(11) DEFAULT NULL,
  `property  `source_key`system` varchar(25511) NOTDEFAULT NULL,
  `property_value` mediumtext  `target_system` varchar(11) DEFAULT NULL,
    `created_date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
    `updated_date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
    PRIMARY KEY (`job_name`,`property_key`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8`name`),
    FOREIGN KEY (owner_email) REFERENCES gobblin_job_owners(email),
    FOREIGN KEY (source_system) REFERENCES gobblin_sync_systems(name),
    FOREIGN KEY (target_system) REFERENCES gobblin_sync_systems(name),
    INDEX (name),
    INDEX (id)
) AUTO_INCREMENT=1;

CREATE TABLE `gobblin_sync_systems_maintenance` (
    `id` int(11) NOT NULL AUTO_INCREMENT,
    `sync_system_name` varchar(255) NOT NULL DEFAULT '',
    `type` varchar(255) NOT NULL DEFAULT '',
    `start_time` datetime NOT NULL,
    `end_time` datetime NOT NULL,
    PRIMARY KEY (`id`),
    FOREIGN KEY (sync_system_name) REFERENCES gobblin_sync_systems(name)
);

Create New job example

Code Block

title	POST body

POST http://localhost:9090/job/PullFromWikipedia

{
	"jobName": "PullFromWikipedia",
	"jobDescription": "A getting started example for Gobblin",
	"jobDisabled": false,
	"jobProperties": {
		"job.group": "Wikipedia",
		"source.class": "org.apache.gobblin.example.wikipedia.WikipediaSource",
		"test_remove": "test_remove_value",
		"something": "asdsad"
	}
}

2. update existing job

Code Block

title	POST body

POST http://localhost:9090/job/PullFromWikipedia

{
	"jobName": "PullFromWikipedia",
	"jobDescription": "A getting started example for Gobblin",
	"jobDisabled": false,
	"jobProperties": {
		"job.group": "Wikipedia",
		"source.class": "org.apache.gobblin.example.wikipedia.WikipediaSource",
		"test_remove": "test_remove_value",
		"something": "asdsad"
	},
	"_jobPropertiesToRemove": [ 
		"somethingkey"	 
	]
}

23. GET existing job

Code Block

title	GET Result

GET http://localhost:9090/job/PullFromWikipedia
{
    "jobName": "PullFromWikipedia",
    "jobId": 14,
    "jobProperties": {
        "source.class": "org.apache.gobblin.example.wikipedia.WikipediaSource",
        "test_remove": "test_remove_value",
        "job.group": "Wikipedia",
        "something": "asdsad"
    },
    "jobDisabled": false,
    "jobDescription": "A getting started example for Gobblin - pull some pages from wikipedia",
    "created_date": 1575937161000,
    "updated_date": 1575937161000
}

...

Page tree

Versions Compared

Old Version 2

New Version 3

Key