2018-11-18 01:39:04 +00:00
|
|
|
/*
|
|
|
|
* This file is part of Background Jobs.
|
|
|
|
*
|
2019-05-25 20:26:12 +00:00
|
|
|
* Copyright © 2019 Riley Trautman
|
2018-11-18 01:39:04 +00:00
|
|
|
*
|
|
|
|
* Background Jobs is free software: you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License as published by
|
|
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
|
|
* (at your option) any later version.
|
|
|
|
*
|
|
|
|
* Background Jobs is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with Background Jobs. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
*/
|
|
|
|
|
|
|
|
//! # Background Jobs
|
|
|
|
//!
|
|
|
|
//! This crate provides tooling required to run some processes asynchronously from a usually
|
|
|
|
//! synchronous application. The standard example of this is Web Services, where certain things
|
|
|
|
//! need to be processed, but processing them while a user is waiting for their browser to respond
|
|
|
|
//! might not be the best experience.
|
|
|
|
//!
|
|
|
|
//! ### Usage
|
|
|
|
//! #### Add Background Jobs to your project
|
|
|
|
//! ```toml
|
|
|
|
//! [dependencies]
|
2019-05-25 21:15:09 +00:00
|
|
|
//! actix = "0.8"
|
2019-09-22 18:13:05 +00:00
|
|
|
//! background-jobs = "0.6.0"
|
2020-03-21 14:44:38 +00:00
|
|
|
//! anyhow = "1.0"
|
2018-11-18 01:39:04 +00:00
|
|
|
//! futures = "0.1"
|
2020-03-21 14:44:38 +00:00
|
|
|
//! serde = { version = "1.0", features = ["derive"] }
|
2019-09-22 18:13:05 +00:00
|
|
|
//! sled = "0.28"
|
2018-11-18 01:39:04 +00:00
|
|
|
//! ```
|
2019-05-25 21:15:09 +00:00
|
|
|
//!
|
2018-11-18 01:39:04 +00:00
|
|
|
//! #### To get started with Background Jobs, first you should define a job.
|
|
|
|
//! Jobs are a combination of the data required to perform an operation, and the logic of that
|
|
|
|
//! operation. They implment the `Job`, `serde::Serialize`, and `serde::DeserializeOwned`.
|
|
|
|
//!
|
|
|
|
//! ```rust,ignore
|
2020-03-21 14:44:38 +00:00
|
|
|
//! use anyhow::Error;
|
2019-05-25 21:15:09 +00:00
|
|
|
//! use background_jobs::Job;
|
2020-03-21 14:44:38 +00:00
|
|
|
//! use futures::future::{ok, Ready};
|
2019-05-25 21:15:09 +00:00
|
|
|
//!
|
2020-03-21 14:44:38 +00:00
|
|
|
//! #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
|
2018-11-18 01:39:04 +00:00
|
|
|
//! pub struct MyJob {
|
|
|
|
//! some_usize: usize,
|
|
|
|
//! other_usize: usize,
|
|
|
|
//! }
|
|
|
|
//!
|
|
|
|
//! impl MyJob {
|
|
|
|
//! pub fn new(some_usize: usize, other_usize: usize) -> Self {
|
|
|
|
//! MyJob {
|
|
|
|
//! some_usize,
|
|
|
|
//! other_usize,
|
|
|
|
//! }
|
|
|
|
//! }
|
|
|
|
//! }
|
|
|
|
//!
|
|
|
|
//! impl Job for MyJob {
|
2019-09-22 18:13:05 +00:00
|
|
|
//! type Processor = MyProcessor; // We'll define this later
|
|
|
|
//! type State = ();
|
2020-03-21 14:44:38 +00:00
|
|
|
//! type Future = Ready<Result<(), Error>>;
|
2019-09-22 18:13:05 +00:00
|
|
|
//!
|
|
|
|
//! fn run(self, _: Self::State) -> Self::Future {
|
2019-05-25 21:15:09 +00:00
|
|
|
//! println!("args: {:?}", self);
|
2018-11-18 01:39:04 +00:00
|
|
|
//!
|
2020-03-21 14:44:38 +00:00
|
|
|
//! ok(())
|
2018-11-18 01:39:04 +00:00
|
|
|
//! }
|
|
|
|
//! }
|
|
|
|
//! ```
|
|
|
|
//!
|
2018-11-18 21:05:03 +00:00
|
|
|
//! The run method for a job takes an additional argument, which is the state the job expects to
|
|
|
|
//! use. The state for all jobs defined in an application must be the same. By default, the state
|
|
|
|
//! is an empty tuple, but it's likely you'll want to pass in some Actix address, or something
|
|
|
|
//! else.
|
|
|
|
//!
|
|
|
|
//! Let's re-define the job to care about some application state.
|
|
|
|
//!
|
|
|
|
//! ```rust,ignore
|
2020-03-21 14:44:38 +00:00
|
|
|
//! use anyhow::Error;
|
|
|
|
//!
|
2018-11-18 21:05:03 +00:00
|
|
|
//! #[derive(Clone, Debug)]
|
|
|
|
//! pub struct MyState {
|
|
|
|
//! pub app_name: String,
|
|
|
|
//! }
|
|
|
|
//!
|
2019-05-25 21:15:09 +00:00
|
|
|
//! impl MyState {
|
|
|
|
//! pub fn new(app_name: &str) -> Self {
|
|
|
|
//! MyState {
|
|
|
|
//! app_name: app_name.to_owned(),
|
|
|
|
//! }
|
|
|
|
//! }
|
|
|
|
//! }
|
|
|
|
//!
|
2019-09-22 17:49:28 +00:00
|
|
|
//! impl Job for MyJob {
|
|
|
|
//! type Processor = MyProcessor;
|
|
|
|
//! type State = MyState;
|
2020-03-21 14:44:38 +00:00
|
|
|
//! type Future = Ready<Result<(), Error>>;
|
2019-09-22 17:49:28 +00:00
|
|
|
//!
|
|
|
|
//! fn run(self, state: Self::State) -> Self::Future {
|
2018-11-18 21:05:03 +00:00
|
|
|
//! info!("{}: args, {:?}", state.app_name, self);
|
|
|
|
//!
|
2020-03-21 14:44:38 +00:00
|
|
|
//! ok(())
|
2018-11-18 21:05:03 +00:00
|
|
|
//! }
|
|
|
|
//! }
|
|
|
|
//! ```
|
|
|
|
//!
|
2018-11-18 01:39:04 +00:00
|
|
|
//! #### Next, define a Processor.
|
|
|
|
//! Processors are types that define default attributes for jobs, as well as containing some logic
|
|
|
|
//! used internally to perform the job. Processors must implement `Proccessor` and `Clone`.
|
|
|
|
//!
|
|
|
|
//! ```rust,ignore
|
2019-05-25 21:15:09 +00:00
|
|
|
//! use background_jobs::{Backoff, MaxRetries, Processor};
|
|
|
|
//!
|
|
|
|
//! const DEFAULT_QUEUE: &'static str = "default";
|
|
|
|
//!
|
2018-11-18 01:39:04 +00:00
|
|
|
//! #[derive(Clone, Debug)]
|
|
|
|
//! pub struct MyProcessor;
|
|
|
|
//!
|
2019-09-22 17:49:28 +00:00
|
|
|
//! impl Processor for MyProcessor {
|
2018-11-18 01:39:04 +00:00
|
|
|
//! // The kind of job this processor should execute
|
|
|
|
//! type Job = MyJob;
|
|
|
|
//!
|
|
|
|
//! // The name of the processor. It is super important that each processor has a unique name,
|
|
|
|
//! // because otherwise one processor will overwrite another processor when they're being
|
|
|
|
//! // registered.
|
2019-05-25 21:15:09 +00:00
|
|
|
//! const NAME: &'static str = "MyProcessor";
|
2018-11-18 01:39:04 +00:00
|
|
|
//!
|
|
|
|
//! // The queue that this processor belongs to
|
|
|
|
//! //
|
|
|
|
//! // Workers have the option to subscribe to specific queues, so this is important to
|
|
|
|
//! // determine which worker will call the processor
|
|
|
|
//! //
|
|
|
|
//! // Jobs can optionally override the queue they're spawned on
|
2019-05-25 21:15:09 +00:00
|
|
|
//! const QUEUE: &'static str = DEFAULT_QUEUE;
|
2018-11-18 01:39:04 +00:00
|
|
|
//!
|
|
|
|
//! // The number of times background-jobs should try to retry a job before giving up
|
|
|
|
//! //
|
|
|
|
//! // Jobs can optionally override this value
|
2018-11-18 14:43:10 +00:00
|
|
|
//! const MAX_RETRIES: MaxRetries = MaxRetries::Count(1);
|
2018-11-18 01:39:04 +00:00
|
|
|
//!
|
|
|
|
//! // The logic to determine how often to retry this job if it fails
|
|
|
|
//! //
|
|
|
|
//! // Jobs can optionally override this value
|
2018-11-18 14:43:10 +00:00
|
|
|
//! const BACKOFF_STRATEGY: Backoff = Backoff::Exponential(2);
|
2018-11-18 01:39:04 +00:00
|
|
|
//! }
|
|
|
|
//! ```
|
|
|
|
//!
|
|
|
|
//! #### Running jobs
|
2019-05-25 21:15:09 +00:00
|
|
|
//! By default, this crate ships with the `background-jobs-actix` feature enabled. This uses the
|
|
|
|
//! `background-jobs-actix` crate to spin up a Server and Workers, and provides a mechanism for
|
2018-11-18 01:39:04 +00:00
|
|
|
//! spawning new jobs.
|
|
|
|
//!
|
2019-05-25 21:15:09 +00:00
|
|
|
//! `background-jobs-actix` on it's own doesn't have a mechanism for storing worker state. This
|
|
|
|
//! can be implemented manually by implementing the `Storage` trait from `background-jobs-core`,
|
|
|
|
//! or the `background-jobs-sled-storage` crate can be used to provide a
|
|
|
|
//! [Sled](https://github.com/spacejam/sled)-backed jobs store.
|
2018-11-18 14:43:10 +00:00
|
|
|
//!
|
|
|
|
//! With that out of the way, back to the examples:
|
|
|
|
//!
|
2019-05-25 21:15:09 +00:00
|
|
|
//! ##### Main
|
2018-11-18 01:39:04 +00:00
|
|
|
//! ```rust,ignore
|
2020-03-21 14:44:38 +00:00
|
|
|
//! use anyhow::Error;
|
2019-09-22 17:49:28 +00:00
|
|
|
//! use background_jobs::{ServerConfig, sled_storage::Storage, WorkerConfig};
|
|
|
|
//! use sled::Db;
|
2018-11-18 01:39:04 +00:00
|
|
|
//!
|
2020-03-21 14:44:38 +00:00
|
|
|
//! #[actix_rt::main]
|
|
|
|
//! async fn main() -> Result<(), Error> {
|
2019-05-25 21:15:09 +00:00
|
|
|
//! // Set up our Storage
|
|
|
|
//! let db = Db::start_default("my-sled-db")?;
|
2019-09-22 17:49:28 +00:00
|
|
|
//! let storage = Storage::new(db)?;
|
2018-11-18 01:39:04 +00:00
|
|
|
//!
|
2019-05-25 21:15:09 +00:00
|
|
|
//! // Start the application server. This guards access to to the jobs store
|
|
|
|
//! let queue_handle = ServerConfig::new(storage).start();
|
2018-11-18 01:39:04 +00:00
|
|
|
//!
|
2019-05-25 21:15:09 +00:00
|
|
|
//! // Configure and start our workers
|
2019-09-22 17:49:28 +00:00
|
|
|
//! WorkerConfig::new(move || MyState::new("My App"))
|
|
|
|
//! .register(MyProcessor)
|
|
|
|
//! .set_processor_count(DEFAULT_QUEUE, 16)
|
|
|
|
//! .start(queue_handle.clone());
|
2018-11-18 01:39:04 +00:00
|
|
|
//!
|
2019-05-25 21:15:09 +00:00
|
|
|
//! // Queue our jobs
|
|
|
|
//! queue_handle.queue::<MyProcessor>(MyJob::new(1, 2))?;
|
|
|
|
//! queue_handle.queue::<MyProcessor>(MyJob::new(3, 4))?;
|
|
|
|
//! queue_handle.queue::<MyProcessor>(MyJob::new(5, 6))?;
|
2018-11-18 01:39:04 +00:00
|
|
|
//!
|
2019-05-25 21:15:09 +00:00
|
|
|
//! // Block on Actix
|
2020-03-21 14:44:38 +00:00
|
|
|
//! actix_rt::signal::ctrl_c().await?;
|
2018-11-18 01:39:04 +00:00
|
|
|
//! Ok(())
|
|
|
|
//! }
|
|
|
|
//! ```
|
|
|
|
//!
|
2018-11-18 14:43:10 +00:00
|
|
|
//! ##### Complete Example
|
|
|
|
//! For the complete example project, see
|
2019-05-25 21:15:09 +00:00
|
|
|
//! [the examples folder](https://git.asonix.dog/Aardwolf/background-jobs/src/branch/master/examples/actix-example)
|
2018-12-16 19:44:25 +00:00
|
|
|
//!
|
|
|
|
//! #### Bringing your own server/worker implementation
|
2018-11-18 01:39:04 +00:00
|
|
|
//! If you want to create your own jobs processor based on this idea, you can depend on the
|
2019-05-25 21:15:09 +00:00
|
|
|
//! `background-jobs-core` crate, which provides the Processor and Job traits, as well as some
|
|
|
|
//! other useful types for implementing a jobs processor and job store.
|
2018-11-18 01:39:04 +00:00
|
|
|
|
2019-05-28 00:01:21 +00:00
|
|
|
pub use background_jobs_core::{
|
|
|
|
memory_storage, Backoff, Job, JobStat, MaxRetries, Processor, Stats,
|
|
|
|
};
|
2018-11-07 02:01:43 +00:00
|
|
|
|
2018-12-16 18:43:44 +00:00
|
|
|
#[cfg(feature = "background-jobs-actix")]
|
2020-03-21 02:31:03 +00:00
|
|
|
pub use background_jobs_actix::{create_server, QueueHandle, WorkerConfig};
|
2019-05-25 20:22:26 +00:00
|
|
|
|
|
|
|
#[cfg(feature = "background-jobs-sled-storage")]
|
2019-05-28 00:01:21 +00:00
|
|
|
pub mod sled_storage {
|
|
|
|
pub use background_jobs_sled_storage::{Error, SledStorage as Storage};
|
|
|
|
}
|