From 3519d7c86a6e87584d25f3292b53d3ce865a659e Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Mon, 15 Mar 2010 15:31:37 +0000 Subject: [PATCH] xfrm: flow cache2 --- include/net/flow.h | 39 ++++- include/net/netns/xfrm.h | 4 + include/net/xfrm.h | 1 + net/core/flow.c | 342 ++++++++++++++++++-------------------- net/ipv6/inet6_connection_sock.c | 6 +- net/xfrm/xfrm_policy.c | 271 +++++++++++++++++++++--------- 6 files changed, 394 insertions(+), 269 deletions(-) diff --git a/include/net/flow.h b/include/net/flow.h index 809970b..814a9d2 100644 --- a/include/net/flow.h +++ b/include/net/flow.h @@ -8,6 +8,9 @@ #define _NET_FLOW_H #include +#include +#include +#include #include struct flowi { @@ -86,13 +89,37 @@ struct flowi { struct net; struct sock; -typedef int (*flow_resolve_t)(struct net *net, struct flowi *key, u16 family, - u8 dir, void **objp, atomic_t **obj_refp); -extern void *flow_cache_lookup(struct net *net, struct flowi *key, u16 family, - u8 dir, flow_resolve_t resolver); -extern void flow_cache_flush(void); -extern atomic_t flow_cache_genid; +struct flow_cache_percpu; +struct flow_cache_entry; + +struct flow_cache { + u32 hash_shift; + u32 order; + struct flow_cache_percpu * percpu; + struct notifier_block hotcpu_notifier; + int low_watermark; + int high_watermark; + struct timer_list rnd_timer; + struct kmem_cache * flow_cachep; +}; + +struct flow_cache_entry { + struct flow_cache_entry *next; + struct flowi key; + u16 family; + u8 dir; +}; + +extern struct flow_cache_entry *flow_cache_lookup( + struct flow_cache *cache, struct flowi *key, + u16 family, u8 dir); +extern void flow_cache_entry_put(struct flow_cache_entry *fce); + +void flow_cache_flush(struct flow_cache *fc, + void (*flush)(struct flow_cache *fc, struct flow_cache_entry *fce)); +extern int flow_cache_init(struct flow_cache *cache, size_t entry_size); +extern void flow_cache_fini(struct flow_cache *cache); static inline int flow_cache_uli_match(struct flowi *fl1, struct flowi *fl2) { diff --git a/include/net/netns/xfrm.h b/include/net/netns/xfrm.h index 1ba9127..4bb72c4 100644 --- a/include/net/netns/xfrm.h +++ b/include/net/netns/xfrm.h @@ -41,6 +41,10 @@ struct netns_xfrm { struct xfrm_policy_hash policy_bydst[XFRM_POLICY_MAX * 2]; unsigned int policy_count[XFRM_POLICY_MAX * 2]; struct work_struct policy_hash_work; + atomic_t policy_genid; + struct hlist_head policy_gc_list; + struct work_struct policy_gc_work; + struct flow_cache flow_cache; struct sock *nlsk; diff --git a/include/net/xfrm.h b/include/net/xfrm.h index 223e90a..5cd4e29 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -487,6 +487,7 @@ struct xfrm_policy struct xfrm_lifetime_cfg lft; struct xfrm_lifetime_cur curlft; struct dst_entry *bundles; + atomic_t bundles_genid; struct xfrm_policy_walk_entry walk; u8 type; u8 action; diff --git a/net/core/flow.c b/net/core/flow.c index 5b27992..e3782c2 100644 --- a/net/core/flow.c +++ b/net/core/flow.c @@ -25,114 +25,85 @@ #include #include -struct flow_cache_entry { - struct flow_cache_entry *next; - u16 family; - u8 dir; - u32 genid; - struct flowi key; - void *object; - atomic_t *object_ref; -}; - -atomic_t flow_cache_genid = ATOMIC_INIT(0); - -static u32 flow_hash_shift; -#define flow_hash_size (1 << flow_hash_shift) -static DEFINE_PER_CPU(struct flow_cache_entry **, flow_tables); - -#define flow_table(cpu) (per_cpu(flow_tables, cpu)) - -static struct kmem_cache *flow_cachep __read_mostly; -static int flow_lwm, flow_hwm; - -struct flow_percpu_info { - int hash_rnd_recalc; - u32 hash_rnd; - int count; +struct flow_cache_percpu { + struct flow_cache_entry ** hash_table; + int hash_count; + u32 hash_rnd; + int hash_rnd_recalc; + struct tasklet_struct flush_tasklet; }; -static DEFINE_PER_CPU(struct flow_percpu_info, flow_hash_info); - -#define flow_hash_rnd_recalc(cpu) \ - (per_cpu(flow_hash_info, cpu).hash_rnd_recalc) -#define flow_hash_rnd(cpu) \ - (per_cpu(flow_hash_info, cpu).hash_rnd) -#define flow_count(cpu) \ - (per_cpu(flow_hash_info, cpu).count) - -static struct timer_list flow_hash_rnd_timer; - -#define FLOW_HASH_RND_PERIOD (10 * 60 * HZ) struct flow_flush_info { - atomic_t cpuleft; - struct completion completion; + void (*flush)(struct flow_cache *fc, struct flow_cache_entry *fce); + struct flow_cache * cache; + atomic_t cpuleft; + struct completion completion; }; -static DEFINE_PER_CPU(struct tasklet_struct, flow_flush_tasklets); -#define flow_flush_tasklet(cpu) (&per_cpu(flow_flush_tasklets, cpu)) +#define flow_cache_hash_size(cache) (1 << (cache)->hash_shift) +#define FLOW_HASH_RND_PERIOD (10 * 60 * HZ) static void flow_cache_new_hashrnd(unsigned long arg) { + struct flow_cache *fc = (struct flow_cache *) arg; int i; for_each_possible_cpu(i) - flow_hash_rnd_recalc(i) = 1; + per_cpu_ptr(fc->percpu, i)->hash_rnd_recalc = 1; - flow_hash_rnd_timer.expires = jiffies + FLOW_HASH_RND_PERIOD; - add_timer(&flow_hash_rnd_timer); + fc->rnd_timer.expires = jiffies + FLOW_HASH_RND_PERIOD; + add_timer(&fc->rnd_timer); } -static void flow_entry_kill(int cpu, struct flow_cache_entry *fle) -{ - if (fle->object) - atomic_dec(fle->object_ref); - kmem_cache_free(flow_cachep, fle); - flow_count(cpu)--; -} - -static void __flow_cache_shrink(int cpu, int shrink_to) +static void __flow_cache_shrink(struct flow_cache *fc, + struct flow_cache_percpu *fcp, + int shrink_to) { struct flow_cache_entry *fle, **flp; int i; - for (i = 0; i < flow_hash_size; i++) { + for (i = 0; i < flow_cache_hash_size(fc); i++) { int k = 0; - flp = &flow_table(cpu)[i]; + flp = &fcp->hash_table[i]; while ((fle = *flp) != NULL && k < shrink_to) { k++; flp = &fle->next; } while ((fle = *flp) != NULL) { *flp = fle->next; - flow_entry_kill(cpu, fle); + + kmem_cache_free(fc->flow_cachep, fle); + fcp->hash_count--; } } } -static void flow_cache_shrink(int cpu) +static void flow_cache_shrink(struct flow_cache *fc, + struct flow_cache_percpu *fcp) { - int shrink_to = flow_lwm / flow_hash_size; + int shrink_to = fc->low_watermark / flow_cache_hash_size(fc); - __flow_cache_shrink(cpu, shrink_to); + __flow_cache_shrink(fc, fcp, shrink_to); } -static void flow_new_hash_rnd(int cpu) +static void flow_new_hash_rnd(struct flow_cache *fc, + struct flow_cache_percpu *fcp) { - get_random_bytes(&flow_hash_rnd(cpu), sizeof(u32)); - flow_hash_rnd_recalc(cpu) = 0; - - __flow_cache_shrink(cpu, 0); + get_random_bytes(&fcp->hash_rnd, sizeof(u32)); + fcp->hash_rnd_recalc = 0; + __flow_cache_shrink(fc, fcp, 0); } -static u32 flow_hash_code(struct flowi *key, int cpu) +static u32 flow_hash_code(struct flow_cache *fc, + struct flow_cache_percpu *fcp, + struct flowi *key) { u32 *k = (u32 *) key; - return (jhash2(k, (sizeof(*key) / sizeof(u32)), flow_hash_rnd(cpu)) & - (flow_hash_size - 1)); + return (jhash2(k, (sizeof(*key) / sizeof(u32)), fcp->hash_rnd) + & (flow_cache_hash_size(fc) - 1)); } #if (BITS_PER_LONG == 64) @@ -165,128 +136,100 @@ static int flow_key_compare(struct flowi *key1, struct flowi *key2) return 0; } -void *flow_cache_lookup(struct net *net, struct flowi *key, u16 family, u8 dir, - flow_resolve_t resolver) +struct flow_cache_entry *flow_cache_lookup(struct flow_cache *fc, + struct flowi *key, + u16 family, u8 dir) { struct flow_cache_entry *fle, **head; + struct flow_cache_percpu *fcp; unsigned int hash; - int cpu; local_bh_disable(); - cpu = smp_processor_id(); + fcp = per_cpu_ptr(fc->percpu, smp_processor_id()); fle = NULL; /* Packet really early in init? Making flow_cache_init a * pre-smp initcall would solve this. --RR */ - if (!flow_table(cpu)) + if (!fcp->hash_table) goto nocache; - if (flow_hash_rnd_recalc(cpu)) - flow_new_hash_rnd(cpu); - hash = flow_hash_code(key, cpu); + if (fcp->hash_rnd_recalc) + flow_new_hash_rnd(fc, fcp); + + hash = flow_hash_code(fc, fcp, key); - head = &flow_table(cpu)[hash]; + head = &fcp->hash_table[hash]; for (fle = *head; fle; fle = fle->next) { if (fle->family == family && fle->dir == dir && flow_key_compare(key, &fle->key) == 0) { - if (fle->genid == atomic_read(&flow_cache_genid)) { - void *ret = fle->object; - - if (ret) - atomic_inc(fle->object_ref); - local_bh_enable(); - - return ret; - } - break; - } - } - - if (!fle) { - if (flow_count(cpu) > flow_hwm) - flow_cache_shrink(cpu); - - fle = kmem_cache_alloc(flow_cachep, GFP_ATOMIC); - if (fle) { - fle->next = *head; - *head = fle; - fle->family = family; - fle->dir = dir; - memcpy(&fle->key, key, sizeof(*key)); - fle->object = NULL; - flow_count(cpu)++; + return fle; } } -nocache: - { - int err; - void *obj; - atomic_t *obj_ref; - - err = resolver(net, key, family, dir, &obj, &obj_ref); + if (fcp->hash_count > fc->high_watermark) + flow_cache_shrink(fc, fcp); - if (fle && !err) { - fle->genid = atomic_read(&flow_cache_genid); + fle = kmem_cache_zalloc(fc->flow_cachep, GFP_ATOMIC); + if (!fle) + goto nocache; - if (fle->object) - atomic_dec(fle->object_ref); + fle->next = *head; + *head = fle; + fle->family = family; + fle->dir = dir; + memcpy(&fle->key, key, sizeof(*key)); + fcp->hash_count++; + return fle; - fle->object = obj; - fle->object_ref = obj_ref; - if (obj) - atomic_inc(fle->object_ref); - } - local_bh_enable(); +nocache: + local_bh_enable(); + return NULL; +} - if (err) - obj = ERR_PTR(err); - return obj; - } +void flow_cache_entry_put(struct flow_cache_entry *fce) +{ + local_bh_enable(); } static void flow_cache_flush_tasklet(unsigned long data) { - struct flow_flush_info *info = (void *)data; + struct flow_flush_info *info = (void *) data; + struct flow_cache *fc = (void *) info->cache; + struct flow_cache_percpu *fcp; int i; - int cpu; - cpu = smp_processor_id(); - for (i = 0; i < flow_hash_size; i++) { - struct flow_cache_entry *fle; + if (info->flush == NULL) + goto done; - fle = flow_table(cpu)[i]; - for (; fle; fle = fle->next) { - unsigned genid = atomic_read(&flow_cache_genid); - - if (!fle->object || fle->genid == genid) - continue; + fcp = per_cpu_ptr(fc->percpu, smp_processor_id()); + for (i = 0; i < flow_cache_hash_size(fc); i++) { + struct flow_cache_entry *fle; - fle->object = NULL; - atomic_dec(fle->object_ref); - } + fle = fcp->hash_table[i]; + for (; fle; fle = fle->next) + info->flush(fc, fle); } +done: if (atomic_dec_and_test(&info->cpuleft)) complete(&info->completion); } -static void flow_cache_flush_per_cpu(void *) __attribute__((__unused__)); static void flow_cache_flush_per_cpu(void *data) { struct flow_flush_info *info = data; - int cpu; struct tasklet_struct *tasklet; + int cpu; cpu = smp_processor_id(); - - tasklet = flow_flush_tasklet(cpu); - tasklet->data = (unsigned long)info; + tasklet = &per_cpu_ptr(info->cache->percpu, cpu)->flush_tasklet; + tasklet->data = (unsigned long) data; tasklet_schedule(tasklet); } -void flow_cache_flush(void) +void flow_cache_flush(struct flow_cache *fc, + void (*flush)(struct flow_cache *fc, struct flow_cache_entry *fce)) { struct flow_flush_info info; static DEFINE_MUTEX(flow_flush_sem); @@ -294,6 +237,8 @@ void flow_cache_flush(void) /* Don't want cpus going down or up during this. */ get_online_cpus(); mutex_lock(&flow_flush_sem); + info.cache = fc; + info.flush = flush; atomic_set(&info.cpuleft, num_online_cpus()); init_completion(&info.completion); @@ -307,62 +252,99 @@ void flow_cache_flush(void) put_online_cpus(); } -static void __init flow_cache_cpu_prepare(int cpu) +static void __init flow_cache_cpu_prepare(struct flow_cache *fc, + struct flow_cache_percpu *fcp) +{ + fcp->hash_table = (struct flow_cache_entry **) + __get_free_pages(GFP_KERNEL|__GFP_ZERO, fc->order); + fcp->hash_rnd_recalc = 1; + fcp->hash_count = 0; + + tasklet_init(&fcp->flush_tasklet, flow_cache_flush_tasklet, 0); +} + +static int __cpuinit flow_cache_cpu(struct notifier_block *nfb, + unsigned long action, + void *hcpu) +{ + struct flow_cache *fc = container_of(nfb, struct flow_cache, hotcpu_notifier); + int cpu = (unsigned long) hcpu; + struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, cpu); + + switch (action) { + case CPU_UP_PREPARE: + case CPU_UP_PREPARE_FROZEN: + flow_cache_cpu_prepare(fc, fcp); + if (!fcp->hash_table) + return NOTIFY_BAD; + break; + case CPU_UP_CANCELED: + case CPU_UP_CANCELED_FROZEN: + case CPU_DEAD: + case CPU_DEAD_FROZEN: + if (fcp->hash_table) { + __flow_cache_shrink(fc, fcp, 0); + free_pages((unsigned long) fcp->hash_table, fc->order); + fcp->hash_table = NULL; + } + break; + } + return NOTIFY_OK; +} + +int flow_cache_init(struct flow_cache *fc, size_t entry_size) { - struct tasklet_struct *tasklet; unsigned long order; + int i, r; + + BUG_ON(entry_size < sizeof(struct flow_cache_entry)); + fc->flow_cachep = kmem_cache_create("flow_cache", + entry_size, + 0, SLAB_PANIC, + NULL); + fc->hash_shift = 10; + fc->low_watermark = 2 * flow_cache_hash_size(fc); + fc->high_watermark = 4 * flow_cache_hash_size(fc); + fc->percpu = alloc_percpu(struct flow_cache_percpu); for (order = 0; (PAGE_SIZE << order) < - (sizeof(struct flow_cache_entry *)*flow_hash_size); + (sizeof(struct flow_cache_entry *) * flow_cache_hash_size(fc)); order++) /* NOTHING */; + fc->order = order; - flow_table(cpu) = (struct flow_cache_entry **) - __get_free_pages(GFP_KERNEL|__GFP_ZERO, order); - if (!flow_table(cpu)) - panic("NET: failed to allocate flow cache order %lu\n", order); + setup_timer(&fc->rnd_timer, flow_cache_new_hashrnd, (unsigned long) fc); + fc->rnd_timer.expires = jiffies + FLOW_HASH_RND_PERIOD; + add_timer(&fc->rnd_timer); - flow_hash_rnd_recalc(cpu) = 1; - flow_count(cpu) = 0; + for_each_online_cpu(i) { + r = flow_cache_cpu(&fc->hotcpu_notifier, + CPU_UP_PREPARE, (void*) i); + if (r != NOTIFY_OK) + panic("NET: failed to allocate flow cache order %lu\n", order); + } - tasklet = flow_flush_tasklet(cpu); - tasklet_init(tasklet, flow_cache_flush_tasklet, 0); -} + fc->hotcpu_notifier = (struct notifier_block){ + .notifier_call = flow_cache_cpu, + }; + register_hotcpu_notifier(&fc->hotcpu_notifier); -static int flow_cache_cpu(struct notifier_block *nfb, - unsigned long action, - void *hcpu) -{ - if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) - __flow_cache_shrink((unsigned long)hcpu, 0); - return NOTIFY_OK; + return 0; } -static int __init flow_cache_init(void) +void flow_cache_fini(struct flow_cache *fc) { int i; - flow_cachep = kmem_cache_create("flow_cache", - sizeof(struct flow_cache_entry), - 0, SLAB_PANIC, - NULL); - flow_hash_shift = 10; - flow_lwm = 2 * flow_hash_size; - flow_hwm = 4 * flow_hash_size; - - setup_timer(&flow_hash_rnd_timer, flow_cache_new_hashrnd, 0); - flow_hash_rnd_timer.expires = jiffies + FLOW_HASH_RND_PERIOD; - add_timer(&flow_hash_rnd_timer); + del_timer(&fc->rnd_timer); + unregister_hotcpu_notifier(&fc->hotcpu_notifier); for_each_possible_cpu(i) - flow_cache_cpu_prepare(i); + flow_cache_cpu(&fc->hotcpu_notifier, CPU_DEAD, (void*) i); - hotcpu_notifier(flow_cache_cpu, 0); - return 0; + free_percpu(fc->percpu); + kmem_cache_destroy(fc->flow_cachep); } -module_init(flow_cache_init); - -EXPORT_SYMBOL(flow_cache_genid); EXPORT_SYMBOL(flow_cache_lookup); diff --git a/net/ipv6/inet6_connection_sock.c b/net/ipv6/inet6_connection_sock.c index cc4797d..399853e 100644 --- a/net/ipv6/inet6_connection_sock.c +++ b/net/ipv6/inet6_connection_sock.c @@ -151,8 +151,9 @@ void __inet6_csk_dst_store(struct sock *sk, struct dst_entry *dst, #ifdef CONFIG_XFRM { + struct net *net = sock_net(sk); struct rt6_info *rt = (struct rt6_info *)dst; - rt->rt6i_flow_cache_genid = atomic_read(&flow_cache_genid); + rt->rt6i_flow_cache_genid = atomic_read(&net->xfrm.policy_genid); } #endif } @@ -166,8 +167,9 @@ struct dst_entry *__inet6_csk_dst_check(struct sock *sk, u32 cookie) #ifdef CONFIG_XFRM if (dst) { + struct net *net = sock_net(sk); struct rt6_info *rt = (struct rt6_info *)dst; - if (rt->rt6i_flow_cache_genid != atomic_read(&flow_cache_genid)) { + if (rt->rt6i_flow_cache_genid != atomic_read(&net->xfrm.policy_genid)) { sk->sk_dst_cache = NULL; dst_release(dst); dst = NULL; diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index cb81ca3..82b01c3 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -44,7 +44,6 @@ static struct xfrm_policy_afinfo *xfrm_policy_afinfo[NPROTO]; static struct kmem_cache *xfrm_dst_cache __read_mostly; -static HLIST_HEAD(xfrm_policy_gc_list); static DEFINE_SPINLOCK(xfrm_policy_gc_lock); static struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family); @@ -53,6 +52,7 @@ static void xfrm_init_pmtu(struct dst_entry *dst); static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol, int dir); +static int stale_bundle(struct dst_entry *dst); static inline int __xfrm4_selector_match(struct xfrm_selector *sel, struct flowi *fl) @@ -216,6 +216,35 @@ expired: xfrm_pol_put(xp); } +struct xfrm_flow_cache_entry { + struct flow_cache_entry fce; + struct xfrm_policy *policy; + struct xfrm_dst *dst; + u32 policy_genid, bundles_genid; +}; +#define XFRM_CACHE_NO_POLICY ((struct xfrm_policy *) -1) + +void xfrm_flow_cache_entry_validate(struct flow_cache *fc, + struct flow_cache_entry *fce) +{ + struct net *net = container_of(fc, struct net, xfrm.flow_cache); + struct xfrm_flow_cache_entry *xfc = + container_of(fce, struct xfrm_flow_cache_entry, fce); + + if (xfc->policy_genid != atomic_read(&net->xfrm.policy_genid)) + goto invalid; + if (xfc->policy == NULL || xfc->policy == XFRM_CACHE_NO_POLICY) + return; + if (xfc->policy->walk.dead) + goto invalid; + if (xfc->bundles_genid != atomic_read(&xfc->policy->bundles_genid)) + goto invalid_dst; + return; +invalid: + xfc->policy = NULL; +invalid_dst: + xfc->dst = NULL; +} /* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2 * SPD calls. @@ -269,27 +298,26 @@ static void xfrm_policy_gc_kill(struct xfrm_policy *policy) if (del_timer(&policy->timer)) atomic_dec(&policy->refcnt); - if (atomic_read(&policy->refcnt) > 1) - flow_cache_flush(); - xfrm_pol_put(policy); } static void xfrm_policy_gc_task(struct work_struct *work) { + struct net *net = container_of(work, struct net, xfrm.policy_gc_work); struct xfrm_policy *policy; struct hlist_node *entry, *tmp; struct hlist_head gc_list; spin_lock_bh(&xfrm_policy_gc_lock); - gc_list.first = xfrm_policy_gc_list.first; - INIT_HLIST_HEAD(&xfrm_policy_gc_list); + gc_list.first = net->xfrm.policy_gc_list.first; + INIT_HLIST_HEAD(&net->xfrm.policy_gc_list); spin_unlock_bh(&xfrm_policy_gc_lock); + flow_cache_flush(&net->xfrm.flow_cache, xfrm_flow_cache_entry_validate); + hlist_for_each_entry_safe(policy, entry, tmp, &gc_list, bydst) xfrm_policy_gc_kill(policy); } -static DECLARE_WORK(xfrm_policy_gc_work, xfrm_policy_gc_task); /* Rule must be locked. Release descentant resources, announce * entry dead. The rule must be unlinked from lists to the moment. @@ -297,6 +325,7 @@ static DECLARE_WORK(xfrm_policy_gc_work, xfrm_policy_gc_task); static void xfrm_policy_kill(struct xfrm_policy *policy) { + struct net *net = xp_net(policy); int dead; write_lock_bh(&policy->lock); @@ -310,10 +339,10 @@ static void xfrm_policy_kill(struct xfrm_policy *policy) } spin_lock_bh(&xfrm_policy_gc_lock); - hlist_add_head(&policy->bydst, &xfrm_policy_gc_list); + hlist_add_head(&policy->bydst, &net->xfrm.policy_gc_list); spin_unlock_bh(&xfrm_policy_gc_lock); - schedule_work(&xfrm_policy_gc_work); + schedule_work(&net->xfrm.policy_gc_work); } static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024; @@ -586,7 +615,7 @@ int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl) hlist_add_head(&policy->bydst, chain); xfrm_pol_hold(policy); net->xfrm.policy_count[dir]++; - atomic_inc(&flow_cache_genid); + atomic_inc(&net->xfrm.policy_genid); if (delpol) __xfrm_policy_unlink(delpol, dir); policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir); @@ -619,11 +648,13 @@ int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl) gc_list = dst; policy->bundles = NULL; + atomic_inc(&policy->bundles_genid); } write_unlock(&policy->lock); } read_unlock_bh(&xfrm_policy_lock); + flow_cache_flush(&net->xfrm.flow_cache, NULL); while (gc_list) { struct dst_entry *dst = gc_list; @@ -669,7 +700,7 @@ struct xfrm_policy *xfrm_policy_bysel_ctx(struct net *net, u8 type, int dir, write_unlock_bh(&xfrm_policy_lock); if (ret && delete) { - atomic_inc(&flow_cache_genid); + atomic_inc(&net->xfrm.policy_genid); xfrm_policy_kill(ret); } return ret; @@ -710,7 +741,7 @@ struct xfrm_policy *xfrm_policy_byid(struct net *net, u8 type, int dir, u32 id, write_unlock_bh(&xfrm_policy_lock); if (ret && delete) { - atomic_inc(&flow_cache_genid); + atomic_inc(&net->xfrm.policy_genid); xfrm_policy_kill(ret); } return ret; @@ -824,7 +855,7 @@ int xfrm_policy_flush(struct net *net, u8 type, struct xfrm_audit *audit_info) } } - atomic_inc(&flow_cache_genid); + atomic_inc(&net->xfrm.policy_genid); out: write_unlock_bh(&xfrm_policy_lock); return err; @@ -977,32 +1008,18 @@ fail: return ret; } -static int xfrm_policy_lookup(struct net *net, struct flowi *fl, u16 family, - u8 dir, void **objp, atomic_t **obj_refp) +static struct xfrm_policy *xfrm_policy_lookup( + struct net *net, struct flowi *fl, + u16 family, u8 dir) { +#ifdef CONFIG_XFRM_SUB_POLICY struct xfrm_policy *pol; - int err = 0; -#ifdef CONFIG_XFRM_SUB_POLICY pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family, dir); - if (IS_ERR(pol)) { - err = PTR_ERR(pol); - pol = NULL; - } - if (pol || err) - goto end; + if (pol != NULL) + return pol; #endif - pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN, fl, family, dir); - if (IS_ERR(pol)) { - err = PTR_ERR(pol); - pol = NULL; - } -#ifdef CONFIG_XFRM_SUB_POLICY -end: -#endif - if ((*objp = (void *) pol) != NULL) - *obj_refp = &pol->refcnt; - return err; + return xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN, fl, family, dir); } static inline int policy_to_flow_dir(int dir) @@ -1083,12 +1100,14 @@ static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol, int xfrm_policy_delete(struct xfrm_policy *pol, int dir) { + struct net *net = xp_net(pol); + write_lock_bh(&xfrm_policy_lock); pol = __xfrm_policy_unlink(pol, dir); write_unlock_bh(&xfrm_policy_lock); if (pol) { if (dir < XFRM_POLICY_MAX) - atomic_inc(&flow_cache_genid); + atomic_inc(&net->xfrm.policy_genid); xfrm_policy_kill(pol); return 0; } @@ -1512,13 +1531,34 @@ xfrm_dst_update_origin(struct dst_entry *dst, struct flowi *fl) #endif } -static int stale_bundle(struct dst_entry *dst); - /* Main function: finds/creates a bundle for given flow. * * At the moment we eat a raw IP route. Mostly to speed up lookups * on interfaces with disabled IPsec. */ + +static void xfrm_flow_cache_update(struct net *net, struct flowi *key, + u16 family, u8 dir, + struct xfrm_policy *pol, + struct xfrm_dst *dst) +{ + struct flow_cache_entry *fce; + struct xfrm_flow_cache_entry *xf; + + fce = flow_cache_lookup(&net->xfrm.flow_cache, + key, family, dir); + if (fce == NULL) + return; + + xf = container_of(fce, struct xfrm_flow_cache_entry, fce); + xf->policy_genid = atomic_read(&net->xfrm.policy_genid); + xf->policy = pol; + if (dst != NULL) + xf->bundles_genid = atomic_read(&pol->bundles_genid); + xf->dst = dst; + flow_cache_entry_put(fce); +} + int __xfrm_lookup(struct net *net, struct dst_entry **dst_p, struct flowi *fl, struct sock *sk, int flags) { @@ -1537,8 +1577,10 @@ int __xfrm_lookup(struct net *net, struct dst_entry **dst_p, struct flowi *fl, u8 dir = policy_to_flow_dir(XFRM_POLICY_OUT); restart: - genid = atomic_read(&flow_cache_genid); + family = dst_orig->ops->family; + genid = atomic_read(&net->xfrm.policy_genid); policy = NULL; + dst = NULL; for (pi = 0; pi < ARRAY_SIZE(pols); pi++) pols[pi] = NULL; npols = 0; @@ -1555,24 +1597,51 @@ restart: } if (!policy) { + struct flow_cache_entry *fce; + struct xfrm_flow_cache_entry *xf; + /* To accelerate a bit... */ if ((dst_orig->flags & DST_NOXFRM) || !net->xfrm.policy_count[XFRM_POLICY_OUT]) goto nopol; - policy = flow_cache_lookup(net, fl, dst_orig->ops->family, - dir, xfrm_policy_lookup); - err = PTR_ERR(policy); - if (IS_ERR(policy)) { - XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR); - goto dropdst; + fce = flow_cache_lookup(&net->xfrm.flow_cache, + fl, family, dir); + if (fce == NULL) + goto no_cache; + + xf = container_of(fce, struct xfrm_flow_cache_entry, fce); + xfrm_flow_cache_entry_validate(&net->xfrm.flow_cache, fce); + if (xf->policy != NULL) { + policy = xf->policy; + if (policy != XFRM_CACHE_NO_POLICY) + xfrm_pol_hold(policy); + if (xf->dst != NULL) + dst = dst_clone((struct dst_entry *) xf->dst); + } + flow_cache_entry_put(fce); + if (policy == XFRM_CACHE_NO_POLICY) + goto nopol; + if (dst && !xfrm_bundle_ok(policy, (struct xfrm_dst *) dst, fl, family, 0)) { + dst_release(dst); + dst = NULL; } } +no_cache: + if (!policy) { + policy = xfrm_policy_lookup(net, fl, family, dir); + if (!policy) { + xfrm_flow_cache_update( + net, fl, family, dir, + XFRM_CACHE_NO_POLICY, NULL); + goto nopol; + } + } + if (IS_ERR(policy)) { + XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR); + goto dropdst; + } - if (!policy) - goto nopol; - - family = dst_orig->ops->family; pols[0] = policy; npols ++; xfrm_nr += pols[0]->xfrm_nr; @@ -1583,6 +1652,9 @@ restart: policy->curlft.use_time = get_seconds(); + if (dst) + goto dst_found; + switch (policy->action) { default: case XFRM_POLICY_BLOCK: @@ -1593,18 +1665,11 @@ restart: case XFRM_POLICY_ALLOW: #ifndef CONFIG_XFRM_SUB_POLICY - if (policy->xfrm_nr == 0) { - /* Flow passes not transformed. */ - xfrm_pol_put(policy); - return 0; - } + if (policy->xfrm_nr == 0) + goto no_transform; #endif - /* Try to find matching bundle. - * - * LATER: help from flow cache. It is optional, this - * is required only for output policy. - */ + /* Try to find matching bundle the hard way. */ dst = xfrm_find_bundle(fl, policy, family); if (IS_ERR(dst)) { XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLECHECKERROR); @@ -1644,12 +1709,8 @@ restart: * they are searched. See above not-transformed bypass * is surrounded by non-sub policy configuration, too. */ - if (xfrm_nr == 0) { - /* Flow passes not transformed. */ - xfrm_pols_put(pols, npols); - return 0; - } - + if (xfrm_nr == 0) + goto no_transform; #endif nx = xfrm_tmpl_resolve(pols, npols, fl, xfrm, family); @@ -1680,7 +1741,7 @@ restart: goto error; } if (nx == -EAGAIN || - genid != atomic_read(&flow_cache_genid)) { + genid != atomic_read(&net->xfrm.policy_genid)) { xfrm_pols_put(pols, npols); goto restart; } @@ -1691,11 +1752,8 @@ restart: goto error; } } - if (nx == 0) { - /* Flow passes not transformed. */ - xfrm_pols_put(pols, npols); - return 0; - } + if (nx == 0) + goto no_transform; dst = xfrm_bundle_create(policy, xfrm, nx, fl, dst_orig); err = PTR_ERR(dst); @@ -1744,6 +1802,9 @@ restart: dst_hold(dst); write_unlock_bh(&policy->lock); } + xfrm_flow_cache_update(net, fl, family, dir, + policy, (struct xfrm_dst *) dst); +dst_found: *dst_p = dst; dst_release(dst_orig); xfrm_pols_put(pols, npols); @@ -1761,7 +1822,12 @@ nopol: if (flags & XFRM_LOOKUP_ICMP) goto dropdst; return 0; +no_transform: + /* Flow passes not transformed. */ + xfrm_pols_put(pols, npols); + return 0; } + EXPORT_SYMBOL(__xfrm_lookup); int xfrm_lookup(struct net *net, struct dst_entry **dst_p, struct flowi *fl, @@ -1919,10 +1985,35 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, } } - if (!pol) - pol = flow_cache_lookup(net, &fl, family, fl_dir, - xfrm_policy_lookup); - + if (!pol) { + struct flow_cache_entry *fce; + struct xfrm_flow_cache_entry *xf; + + fce = flow_cache_lookup(&net->xfrm.flow_cache, + &fl, family, dir); + if (fce != NULL) { + xf = container_of(fce, struct xfrm_flow_cache_entry, fce); + xfrm_flow_cache_entry_validate(&net->xfrm.flow_cache, fce); + if (xf->policy != NULL) { + pol = xf->policy; + if (pol != XFRM_CACHE_NO_POLICY) + xfrm_pol_hold(pol); + else + pol = NULL; + } else { + pol = xfrm_policy_lookup(net, &fl, family, dir); + if (!IS_ERR(pol)) { + if (pol) + xf->policy = pol; + else + xf->policy = XFRM_CACHE_NO_POLICY; + } + xf->dst = NULL; + xf->policy_genid = atomic_read(&net->xfrm.policy_genid); + } + flow_cache_entry_put(fce); + } + } if (IS_ERR(pol)) { XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR); return 0; @@ -2121,6 +2212,7 @@ static void prune_one_bundle(struct xfrm_policy *pol, int (*func)(struct dst_ent dstp = &dst->next; } } + atomic_inc(&pol->bundles_genid); write_unlock(&pol->lock); } @@ -2148,6 +2240,7 @@ static void xfrm_prune_bundles(struct net *net, int (*func)(struct dst_entry *)) } read_unlock_bh(&xfrm_policy_lock); + flow_cache_flush(&net->xfrm.flow_cache, NULL); while (gc_list) { struct dst_entry *dst = gc_list; gc_list = dst->next; @@ -2428,6 +2521,9 @@ static int __net_init xfrm_policy_init(struct net *net) INIT_LIST_HEAD(&net->xfrm.policy_all); INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize); + INIT_HLIST_HEAD(&net->xfrm.policy_gc_list); + INIT_WORK(&net->xfrm.policy_gc_work, xfrm_policy_gc_task); + flow_cache_init(&net->xfrm.flow_cache, sizeof(struct xfrm_flow_cache_entry)); if (net_eq(net, &init_net)) register_netdevice_notifier(&xfrm_dev_notifier); return 0; @@ -2461,7 +2557,7 @@ static void xfrm_policy_fini(struct net *net) audit_info.sessionid = -1; audit_info.secid = 0; xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, &audit_info); - flush_work(&xfrm_policy_gc_work); + flush_work(&net->xfrm.policy_gc_work); WARN_ON(!list_empty(&net->xfrm.policy_all)); @@ -2479,6 +2575,8 @@ static void xfrm_policy_fini(struct net *net) sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head); WARN_ON(!hlist_empty(net->xfrm.policy_byidx)); xfrm_hash_free(net->xfrm.policy_byidx, sz); + + flow_cache_fini(&net->xfrm.flow_cache); } static int __net_init xfrm_net_init(struct net *net) @@ -2685,8 +2783,9 @@ static int migrate_tmpl_match(struct xfrm_migrate *m, struct xfrm_tmpl *t) static int xfrm_policy_migrate(struct xfrm_policy *pol, struct xfrm_migrate *m, int num_migrate) { + struct net *net = xp_net(pol); struct xfrm_migrate *mp; - struct dst_entry *dst; + struct dst_entry *gc_list = NULL, *tail; int i, j, n = 0; write_lock_bh(&pol->lock); @@ -2711,15 +2810,25 @@ static int xfrm_policy_migrate(struct xfrm_policy *pol, sizeof(pol->xfrm_vec[i].saddr)); pol->xfrm_vec[i].encap_family = mp->new_family; /* flush bundles */ - while ((dst = pol->bundles) != NULL) { - pol->bundles = dst->next; - dst_free(dst); - } + tail = pol->bundles; + while (tail->next) + tail = tail->next; + tail->next = gc_list; + gc_list = pol->bundles; + pol->bundles = NULL; + atomic_inc(&pol->bundles_genid); } } - write_unlock_bh(&pol->lock); + flow_cache_flush(&net->xfrm.flow_cache, NULL); + while (gc_list) { + struct dst_entry *dst = gc_list; + + gc_list = dst->next; + dst_free(dst); + } + if (!n) return -ENODATA; -- 1.7.0.2