asonix/background-jobs
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Update tokio-zmq, optionally use futures-zmq instead

Browse source
This commit is contained in:
asonix 2018-11-16 18:09:53 -06:00
parent a877b5c9e8
commit 960c0235c4
13 changed files with 126 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Cargo.toml
View file

@ -16,7 +16,7 @@ members = [
]
[features]
default = ["jobs-actix", "jobs-server-tokio", "jobs-tokio"]
default = ["jobs-actix", "jobs-server-tokio", "jobs-server-tokio/tokio-zmq", "jobs-tokio"]
[dependencies.jobs-actix]
version = "0.1"

2
examples/server-jobs-example/.env
View file

@ -1 +1 @@
RUST_LOG=info
RUST_LOG=server_jobs_example=info

12
jobs-server-tokio/Cargo.toml
View file

@ -12,9 +12,19 @@ serde = "1.0"
serde_json = "1.0"
tokio = "0.1"
tokio-threadpool = "0.1"
tokio-zmq = "0.6.1"
zmq = "0.8"
[features]
default = ["tokio-zmq"]
[dependencies.jobs-core]
version = "0.1"
path = "../jobs-core"
[dependencies.tokio-zmq]
version = "0.8"
optional = true
[dependencies.futures-zmq]
version = "0.3"
optional = true

5
jobs-server-tokio/src/lib.rs
View file

@ -1,8 +1,3 @@
#[macro_use]
extern crate failure;
#[macro_use]
extern crate log;
use failure::Error;
mod server;

3
jobs-server-tokio/src/server/mod.rs
View file

@ -4,9 +4,10 @@ use std::{
sync::Arc,
};
use failure::Error;
use failure::{Error, Fail};
use futures::{future::poll_fn, Future};
use jobs_core::Storage;
use log::{error, info};
use tokio_threadpool::blocking;
use zmq::Context;

34
jobs-server-tokio/src/server/portmap.rs
View file

@ -1,8 +1,12 @@
use std::{collections::BTreeMap, sync::Arc, time::Duration};
use failure::Error;
use futures::{future::lazy, Future, Stream};
use futures::{Future, Stream};
#[cfg(feature = "futures-zmq")]
use futures_zmq::{prelude::*, Multipart, Rep};
use log::{error, info};
use tokio::timer::Delay;
#[cfg(feature = "tokio-zmq")]
use tokio_zmq::{prelude::*, Multipart, Rep};
use zmq::Message;
@ -85,21 +89,19 @@ impl ResetPortMapConfig {
}
fn build(self) -> impl Future<Item = (), Error = Error> {
lazy(|| {
let rep = Rep::builder(self.config.context.clone())
.bind(&self.address)
.build()?;
Rep::builder(self.config.context.clone())
.bind(&self.address)
.build()
.map(|rep| {
let config = PortMapConfig {
rep,
address: self.address,
port_map: self.port_map,
config: self.config,
};
let config = PortMapConfig {
rep,
address: self.address,
port_map: self.port_map,
config: self.config,
};
tokio::spawn(config.run());
Ok(())
})
tokio::spawn(config.run());
})
.from_err()
}
}

39
jobs-server-tokio/src/server/pull.rs
View file

@ -1,13 +1,14 @@
use std::{sync::Arc, time::Duration};
use failure::Error;
use futures::{
future::{lazy, poll_fn},
Future, Stream,
};
use failure::{Error, Fail};
use futures::{future::poll_fn, Future, Stream};
#[cfg(feature = "futures-zmq")]
use futures_zmq::{prelude::*, Multipart, Pull};
use jobs_core::{JobInfo, Storage};
use log::{error, info, trace};
use tokio::timer::Delay;
use tokio_threadpool::blocking;
#[cfg(feature = "tokio-zmq")]
use tokio_zmq::{prelude::*, Multipart, Pull};
use crate::server::{coerce, Config};
@ -112,21 +113,19 @@ impl ResetPullConfig {
}
fn build(self) -> impl Future<Item = (), Error = Error> {
lazy(|| {
let puller = Pull::builder(self.config.context.clone())
.bind(&self.address)
.build()?;
Pull::builder(self.config.context.clone())
.bind(&self.address)
.build()
.map(|puller| {
let config = PullConfig {
puller,
address: self.address,
storage: self.storage,
config: self.config,
};
let config = PullConfig {
puller,
address: self.address,
storage: self.storage,
config: self.config,
};
tokio::spawn(config.run());
Ok(())
})
tokio::spawn(config.run());
})
.from_err()
}
}

42
jobs-server-tokio/src/server/push.rs
View file

@ -1,14 +1,14 @@
use std::{sync::Arc, time::Duration};
use failure::Error;
use futures::{
future::{lazy, poll_fn},
stream::iter_ok,
Future, Stream,
};
use futures::{future::poll_fn, stream::iter_ok, Future, Stream};
#[cfg(feature = "futures-zmq")]
use futures_zmq::{prelude::*, Multipart, Push};
use jobs_core::{JobInfo, Storage};
use log::{error, info};
use tokio::timer::{Delay, Interval};
use tokio_threadpool::blocking;
#[cfg(feature = "tokio-zmq")]
use tokio_zmq::{prelude::*, Multipart, Push};
use zmq::Message;
@ -136,23 +136,21 @@ impl ResetPushConfig {
}
fn build(self) -> impl Future<Item = (), Error = Error> {
lazy(|| {
info!("Building and spawning new server");
let pusher = Push::builder(self.config.context.clone())
.bind(&self.address)
.build()?;
info!("Building and spawning new server");
Push::builder(self.config.context.clone())
.bind(&self.address)
.build()
.map(|pusher| {
let config = PushConfig {
pusher,
address: self.address,
queue: self.queue,
storage: self.storage,
config: self.config,
};
let config = PushConfig {
pusher,
address: self.address,
queue: self.queue,
storage: self.storage,
config: self.config,
};
tokio::spawn(config.run());
Ok(())
})
tokio::spawn(config.run());
})
.from_err()
}
}

1
jobs-server-tokio/src/server/stalled.rs
View file

@ -3,6 +3,7 @@ use std::{sync::Arc, time::Duration};
use failure::Error;
use futures::{future::poll_fn, Future, Stream};
use jobs_core::Storage;
use log::{error, info};
use tokio::timer::{Delay, Interval};
use tokio_threadpool::blocking;

4
jobs-server-tokio/src/spawner.rs
View file

@ -2,7 +2,11 @@ use std::sync::Arc;
use failure::Error;
use futures::{future::IntoFuture, Future};
#[cfg(feature = "futures-zmq")]
use futures_zmq::{prelude::*, Push};
use jobs_core::JobInfo;
use log::{debug, trace};
#[cfg(feature = "tokio-zmq")]
use tokio_zmq::{prelude::*, Push};
use zmq::{Context, Message};

51
jobs-server-tokio/src/worker/config.rs
View file

@ -1,12 +1,16 @@
use std::{sync::Arc, time::Duration};
use failure::Error;
use failure::{Error, Fail};
use futures::{
future::{lazy, Either, IntoFuture},
future::{Either, IntoFuture},
Future, Stream,
};
#[cfg(feature = "futures-zmq")]
use futures_zmq::{prelude::*, Multipart, Pull, Push};
use jobs_core::{JobInfo, Processors};
use log::{error, info};
use tokio::timer::Delay;
#[cfg(feature = "tokio-zmq")]
use tokio_zmq::{prelude::*, Multipart, Pull, Push};
use zmq::{Context, Message};
@ -98,29 +102,28 @@ impl ResetWorker {
}
fn build(self) -> impl Future<Item = (), Error = Error> {
lazy(|| {
let push = Push::builder(self.context.clone())
.connect(&self.push_address)
.build()?;
Push::builder(self.context.clone())
.connect(&self.push_address)
.build()
.join(
Pull::builder(self.context.clone())
.connect(&self.pull_address)
.build(),
)
.map(|(push, pull)| {
let config = Worker {
push,
pull,
push_address: self.push_address,
pull_address: self.pull_address,
queue: self.queue,
processors: self.processors,
context: self.context,
};
let pull = Pull::builder(self.context.clone())
.connect(&self.pull_address)
.build()?;
let config = Worker {
push,
pull,
push_address: self.push_address,
pull_address: self.pull_address,
queue: self.queue,
processors: self.processors,
context: self.context,
};
tokio::spawn(config.run());
Ok(())
})
tokio::spawn(config.run());
})
.from_err()
}
}

2
jobs-server-tokio/src/worker/mod.rs
View file

@ -1,7 +1,9 @@
use std::{collections::BTreeMap, sync::Arc};
use failure::Fail;
use futures::Future;
use jobs_core::{Processor, Processors};
use log::{error, info};
use zmq::Context;
mod config;

38
jobs-server-tokio/src/worker/portmap.rs
View file

@ -1,7 +1,10 @@
use std::{collections::BTreeMap, sync::Arc};
use failure::Error;
use futures::{future::lazy, Future};
use failure::{Error, Fail};
use futures::Future;
#[cfg(feature = "futures-zmq")]
use futures_zmq::{prelude::*, Req};
#[cfg(feature = "tokio-zmq")]
use tokio_zmq::{prelude::*, Req};
use zmq::{Context, Message};
@ -12,24 +15,23 @@ impl PortMap {
address: String,
context: Arc<Context>,
) -> impl Future<Item = BTreeMap<String, usize>, Error = Error> {
lazy(move || {
let req = Req::builder(context.clone()).connect(&address).build()?;
Req::builder(context.clone())
.connect(&address)
.build()
.from_err()
.and_then(|req| {
Message::from_slice(b"h")
.map_err(Error::from)
.map(move |msg| (req, msg.into()))
})
.and_then(|(req, msg)| req.send(msg).and_then(|req| req.recv()).from_err())
.and_then(|(mut multipart, _)| {
let msg = multipart.pop_front().ok_or(EmptyMessage)?;
Ok(req)
})
.and_then(|req| {
Message::from_slice(b"h")
.map_err(Error::from)
.map(move |msg| (req, msg.into()))
})
.and_then(|(req, msg)| req.send(msg).and_then(|req| req.recv()).from_err())
.and_then(|(mut multipart, _)| {
let msg = multipart.pop_front().ok_or(EmptyMessage)?;
let map = serde_json::from_slice(&msg)?;
let map = serde_json::from_slice(&msg)?;
Ok(map)
})
Ok(map)
})
}
}