IB/sa: Require SA registration

Require users to register with SA module, to prevent the sa_query
module text from going away while an SA query callback is still
running.  Update all in-tree users for the new interface.

Signed-off-by: Michael S. Tsirkin <mst@mellanox.co.il>
Signed-off-by: Sean Hefty <sean.hefty@intel.com>
Signed-off-by: Roland Dreier <rolandd@cisco.com>
This commit is contained in:
Michael S. Tsirkin 2006-08-21 16:40:12 -07:00 committed by Roland Dreier
parent 2439a6e65f
commit c1a0b23bf4
7 changed files with 116 additions and 27 deletions

View file

@ -62,6 +62,7 @@ static struct ib_client cma_client = {
.remove = cma_remove_one .remove = cma_remove_one
}; };
static struct ib_sa_client sa_client;
static LIST_HEAD(dev_list); static LIST_HEAD(dev_list);
static LIST_HEAD(listen_any_list); static LIST_HEAD(listen_any_list);
static DEFINE_MUTEX(lock); static DEFINE_MUTEX(lock);
@ -1323,7 +1324,7 @@ static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(addr)); path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(addr));
path_rec.numb_path = 1; path_rec.numb_path = 1;
id_priv->query_id = ib_sa_path_rec_get(id_priv->id.device, id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
id_priv->id.port_num, &path_rec, id_priv->id.port_num, &path_rec,
IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID | IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH, IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH,
@ -2199,12 +2200,15 @@ static int cma_init(void)
if (!cma_wq) if (!cma_wq)
return -ENOMEM; return -ENOMEM;
ib_sa_register_client(&sa_client);
ret = ib_register_client(&cma_client); ret = ib_register_client(&cma_client);
if (ret) if (ret)
goto err; goto err;
return 0; return 0;
err: err:
ib_sa_unregister_client(&sa_client);
destroy_workqueue(cma_wq); destroy_workqueue(cma_wq);
return ret; return ret;
} }
@ -2212,6 +2216,7 @@ err:
static void cma_cleanup(void) static void cma_cleanup(void)
{ {
ib_unregister_client(&cma_client); ib_unregister_client(&cma_client);
ib_sa_unregister_client(&sa_client);
destroy_workqueue(cma_wq); destroy_workqueue(cma_wq);
idr_destroy(&sdp_ps); idr_destroy(&sdp_ps);
idr_destroy(&tcp_ps); idr_destroy(&tcp_ps);

View file

@ -1,6 +1,7 @@
/* /*
* Copyright (c) 2004 Topspin Communications. All rights reserved. * Copyright (c) 2004 Topspin Communications. All rights reserved.
* Copyright (c) 2005 Voltaire, Inc.  All rights reserved. * Copyright (c) 2005 Voltaire, Inc.  All rights reserved.
* Copyright (c) 2006 Intel Corporation. All rights reserved.
* *
* This software is available to you under a choice of one of two * This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU * licenses. You may choose to be licensed under the terms of the GNU
@ -75,6 +76,7 @@ struct ib_sa_device {
struct ib_sa_query { struct ib_sa_query {
void (*callback)(struct ib_sa_query *, int, struct ib_sa_mad *); void (*callback)(struct ib_sa_query *, int, struct ib_sa_mad *);
void (*release)(struct ib_sa_query *); void (*release)(struct ib_sa_query *);
struct ib_sa_client *client;
struct ib_sa_port *port; struct ib_sa_port *port;
struct ib_mad_send_buf *mad_buf; struct ib_mad_send_buf *mad_buf;
struct ib_sa_sm_ah *sm_ah; struct ib_sa_sm_ah *sm_ah;
@ -415,6 +417,31 @@ static void ib_sa_event(struct ib_event_handler *handler, struct ib_event *event
} }
} }
void ib_sa_register_client(struct ib_sa_client *client)
{
atomic_set(&client->users, 1);
init_completion(&client->comp);
}
EXPORT_SYMBOL(ib_sa_register_client);
static inline void ib_sa_client_get(struct ib_sa_client *client)
{
atomic_inc(&client->users);
}
static inline void ib_sa_client_put(struct ib_sa_client *client)
{
if (atomic_dec_and_test(&client->users))
complete(&client->comp);
}
void ib_sa_unregister_client(struct ib_sa_client *client)
{
ib_sa_client_put(client);
wait_for_completion(&client->comp);
}
EXPORT_SYMBOL(ib_sa_unregister_client);
/** /**
* ib_sa_cancel_query - try to cancel an SA query * ib_sa_cancel_query - try to cancel an SA query
* @id:ID of query to cancel * @id:ID of query to cancel
@ -557,6 +584,7 @@ static void ib_sa_path_rec_release(struct ib_sa_query *sa_query)
/** /**
* ib_sa_path_rec_get - Start a Path get query * ib_sa_path_rec_get - Start a Path get query
* @client:SA client
* @device:device to send query on * @device:device to send query on
* @port_num: port number to send query on * @port_num: port number to send query on
* @rec:Path Record to send in query * @rec:Path Record to send in query
@ -579,7 +607,8 @@ static void ib_sa_path_rec_release(struct ib_sa_query *sa_query)
* error code. Otherwise it is a query ID that can be used to cancel * error code. Otherwise it is a query ID that can be used to cancel
* the query. * the query.
*/ */
int ib_sa_path_rec_get(struct ib_device *device, u8 port_num, int ib_sa_path_rec_get(struct ib_sa_client *client,
struct ib_device *device, u8 port_num,
struct ib_sa_path_rec *rec, struct ib_sa_path_rec *rec,
ib_sa_comp_mask comp_mask, ib_sa_comp_mask comp_mask,
int timeout_ms, gfp_t gfp_mask, int timeout_ms, gfp_t gfp_mask,
@ -614,8 +643,10 @@ int ib_sa_path_rec_get(struct ib_device *device, u8 port_num,
goto err1; goto err1;
} }
query->callback = callback; ib_sa_client_get(client);
query->context = context; query->sa_query.client = client;
query->callback = callback;
query->context = context;
mad = query->sa_query.mad_buf->mad; mad = query->sa_query.mad_buf->mad;
init_mad(mad, agent); init_mad(mad, agent);
@ -639,6 +670,7 @@ int ib_sa_path_rec_get(struct ib_device *device, u8 port_num,
err2: err2:
*sa_query = NULL; *sa_query = NULL;
ib_sa_client_put(query->sa_query.client);
ib_free_send_mad(query->sa_query.mad_buf); ib_free_send_mad(query->sa_query.mad_buf);
err1: err1:
@ -671,6 +703,7 @@ static void ib_sa_service_rec_release(struct ib_sa_query *sa_query)
/** /**
* ib_sa_service_rec_query - Start Service Record operation * ib_sa_service_rec_query - Start Service Record operation
* @client:SA client
* @device:device to send request on * @device:device to send request on
* @port_num: port number to send request on * @port_num: port number to send request on
* @method:SA method - should be get, set, or delete * @method:SA method - should be get, set, or delete
@ -695,7 +728,8 @@ static void ib_sa_service_rec_release(struct ib_sa_query *sa_query)
* error code. Otherwise it is a request ID that can be used to cancel * error code. Otherwise it is a request ID that can be used to cancel
* the query. * the query.
*/ */
int ib_sa_service_rec_query(struct ib_device *device, u8 port_num, u8 method, int ib_sa_service_rec_query(struct ib_sa_client *client,
struct ib_device *device, u8 port_num, u8 method,
struct ib_sa_service_rec *rec, struct ib_sa_service_rec *rec,
ib_sa_comp_mask comp_mask, ib_sa_comp_mask comp_mask,
int timeout_ms, gfp_t gfp_mask, int timeout_ms, gfp_t gfp_mask,
@ -735,8 +769,10 @@ int ib_sa_service_rec_query(struct ib_device *device, u8 port_num, u8 method,
goto err1; goto err1;
} }
query->callback = callback; ib_sa_client_get(client);
query->context = context; query->sa_query.client = client;
query->callback = callback;
query->context = context;
mad = query->sa_query.mad_buf->mad; mad = query->sa_query.mad_buf->mad;
init_mad(mad, agent); init_mad(mad, agent);
@ -761,6 +797,7 @@ int ib_sa_service_rec_query(struct ib_device *device, u8 port_num, u8 method,
err2: err2:
*sa_query = NULL; *sa_query = NULL;
ib_sa_client_put(query->sa_query.client);
ib_free_send_mad(query->sa_query.mad_buf); ib_free_send_mad(query->sa_query.mad_buf);
err1: err1:
@ -791,7 +828,8 @@ static void ib_sa_mcmember_rec_release(struct ib_sa_query *sa_query)
kfree(container_of(sa_query, struct ib_sa_mcmember_query, sa_query)); kfree(container_of(sa_query, struct ib_sa_mcmember_query, sa_query));
} }
int ib_sa_mcmember_rec_query(struct ib_device *device, u8 port_num, int ib_sa_mcmember_rec_query(struct ib_sa_client *client,
struct ib_device *device, u8 port_num,
u8 method, u8 method,
struct ib_sa_mcmember_rec *rec, struct ib_sa_mcmember_rec *rec,
ib_sa_comp_mask comp_mask, ib_sa_comp_mask comp_mask,
@ -827,8 +865,10 @@ int ib_sa_mcmember_rec_query(struct ib_device *device, u8 port_num,
goto err1; goto err1;
} }
query->callback = callback; ib_sa_client_get(client);
query->context = context; query->sa_query.client = client;
query->callback = callback;
query->context = context;
mad = query->sa_query.mad_buf->mad; mad = query->sa_query.mad_buf->mad;
init_mad(mad, agent); init_mad(mad, agent);
@ -853,6 +893,7 @@ int ib_sa_mcmember_rec_query(struct ib_device *device, u8 port_num,
err2: err2:
*sa_query = NULL; *sa_query = NULL;
ib_sa_client_put(query->sa_query.client);
ib_free_send_mad(query->sa_query.mad_buf); ib_free_send_mad(query->sa_query.mad_buf);
err1: err1:
@ -889,6 +930,7 @@ static void send_handler(struct ib_mad_agent *agent,
ib_free_send_mad(mad_send_wc->send_buf); ib_free_send_mad(mad_send_wc->send_buf);
kref_put(&query->sm_ah->ref, free_sm_ah); kref_put(&query->sm_ah->ref, free_sm_ah);
ib_sa_client_put(query->client);
query->release(query); query->release(query);
} }

View file

@ -336,6 +336,8 @@ static inline void ipoib_unregister_debugfs(void) { }
extern int ipoib_sendq_size; extern int ipoib_sendq_size;
extern int ipoib_recvq_size; extern int ipoib_recvq_size;
extern struct ib_sa_client ipoib_sa_client;
#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG #ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
extern int ipoib_debug_level; extern int ipoib_debug_level;

View file

@ -82,6 +82,8 @@ static const u8 ipv4_bcast_addr[] = {
struct workqueue_struct *ipoib_workqueue; struct workqueue_struct *ipoib_workqueue;
struct ib_sa_client ipoib_sa_client;
static void ipoib_add_one(struct ib_device *device); static void ipoib_add_one(struct ib_device *device);
static void ipoib_remove_one(struct ib_device *device); static void ipoib_remove_one(struct ib_device *device);
@ -463,7 +465,7 @@ static int path_rec_start(struct net_device *dev,
init_completion(&path->done); init_completion(&path->done);
path->query_id = path->query_id =
ib_sa_path_rec_get(priv->ca, priv->port, ib_sa_path_rec_get(&ipoib_sa_client, priv->ca, priv->port,
&path->pathrec, &path->pathrec,
IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_DGID |
IB_SA_PATH_REC_SGID | IB_SA_PATH_REC_SGID |
@ -1191,13 +1193,16 @@ static int __init ipoib_init_module(void)
goto err_fs; goto err_fs;
} }
ib_sa_register_client(&ipoib_sa_client);
ret = ib_register_client(&ipoib_client); ret = ib_register_client(&ipoib_client);
if (ret) if (ret)
goto err_wq; goto err_sa;
return 0; return 0;
err_wq: err_sa:
ib_sa_unregister_client(&ipoib_sa_client);
destroy_workqueue(ipoib_workqueue); destroy_workqueue(ipoib_workqueue);
err_fs: err_fs:
@ -1209,6 +1214,7 @@ err_fs:
static void __exit ipoib_cleanup_module(void) static void __exit ipoib_cleanup_module(void)
{ {
ib_unregister_client(&ipoib_client); ib_unregister_client(&ipoib_client);
ib_sa_unregister_client(&ipoib_sa_client);
ipoib_unregister_debugfs(); ipoib_unregister_debugfs();
destroy_workqueue(ipoib_workqueue); destroy_workqueue(ipoib_workqueue);
} }

View file

@ -361,7 +361,7 @@ static int ipoib_mcast_sendonly_join(struct ipoib_mcast *mcast)
init_completion(&mcast->done); init_completion(&mcast->done);
ret = ib_sa_mcmember_rec_set(priv->ca, priv->port, &rec, ret = ib_sa_mcmember_rec_set(&ipoib_sa_client, priv->ca, priv->port, &rec,
IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_MGID |
IB_SA_MCMEMBER_REC_PORT_GID | IB_SA_MCMEMBER_REC_PORT_GID |
IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_PKEY |
@ -485,9 +485,9 @@ static void ipoib_mcast_join(struct net_device *dev, struct ipoib_mcast *mcast,
init_completion(&mcast->done); init_completion(&mcast->done);
ret = ib_sa_mcmember_rec_set(priv->ca, priv->port, &rec, comp_mask, ret = ib_sa_mcmember_rec_set(&ipoib_sa_client, priv->ca, priv->port,
mcast->backoff * 1000, GFP_ATOMIC, &rec, comp_mask, mcast->backoff * 1000,
ipoib_mcast_join_complete, GFP_ATOMIC, ipoib_mcast_join_complete,
mcast, &mcast->query); mcast, &mcast->query);
if (ret < 0) { if (ret < 0) {
@ -528,7 +528,7 @@ void ipoib_mcast_join_task(void *dev_ptr)
priv->local_rate = attr.active_speed * priv->local_rate = attr.active_speed *
ib_width_enum_to_int(attr.active_width); ib_width_enum_to_int(attr.active_width);
} else } else
ipoib_warn(priv, "ib_query_port failed\n"); ipoib_warn(priv, "ib_query_port failed\n");
} }
if (!priv->broadcast) { if (!priv->broadcast) {
@ -681,7 +681,7 @@ static int ipoib_mcast_leave(struct net_device *dev, struct ipoib_mcast *mcast)
* Just make one shot at leaving and don't wait for a reply; * Just make one shot at leaving and don't wait for a reply;
* if we fail, too bad. * if we fail, too bad.
*/ */
ret = ib_sa_mcmember_rec_delete(priv->ca, priv->port, &rec, ret = ib_sa_mcmember_rec_delete(&ipoib_sa_client, priv->ca, priv->port, &rec,
IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_MGID |
IB_SA_MCMEMBER_REC_PORT_GID | IB_SA_MCMEMBER_REC_PORT_GID |
IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_PKEY |

View file

@ -96,6 +96,8 @@ static struct ib_client srp_client = {
.remove = srp_remove_one .remove = srp_remove_one
}; };
static struct ib_sa_client srp_sa_client;
static inline struct srp_target_port *host_to_target(struct Scsi_Host *host) static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
{ {
return (struct srp_target_port *) host->hostdata; return (struct srp_target_port *) host->hostdata;
@ -267,7 +269,8 @@ static int srp_lookup_path(struct srp_target_port *target)
init_completion(&target->done); init_completion(&target->done);
target->path_query_id = ib_sa_path_rec_get(target->srp_host->dev->dev, target->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
target->srp_host->dev->dev,
target->srp_host->port, target->srp_host->port,
&target->path, &target->path,
IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_DGID |
@ -1998,9 +2001,12 @@ static int __init srp_init_module(void)
return ret; return ret;
} }
ib_sa_register_client(&srp_sa_client);
ret = ib_register_client(&srp_client); ret = ib_register_client(&srp_client);
if (ret) { if (ret) {
printk(KERN_ERR PFX "couldn't register IB client\n"); printk(KERN_ERR PFX "couldn't register IB client\n");
ib_sa_unregister_client(&srp_sa_client);
class_unregister(&srp_class); class_unregister(&srp_class);
return ret; return ret;
} }
@ -2011,6 +2017,7 @@ static int __init srp_init_module(void)
static void __exit srp_cleanup_module(void) static void __exit srp_cleanup_module(void)
{ {
ib_unregister_client(&srp_client); ib_unregister_client(&srp_client);
ib_sa_unregister_client(&srp_sa_client);
class_unregister(&srp_class); class_unregister(&srp_class);
} }

View file

@ -1,6 +1,7 @@
/* /*
* Copyright (c) 2004 Topspin Communications. All rights reserved. * Copyright (c) 2004 Topspin Communications. All rights reserved.
* Copyright (c) 2005 Voltaire, Inc. All rights reserved. * Copyright (c) 2005 Voltaire, Inc. All rights reserved.
* Copyright (c) 2006 Intel Corporation. All rights reserved.
* *
* This software is available to you under a choice of one of two * This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU * licenses. You may choose to be licensed under the terms of the GNU
@ -36,8 +37,11 @@
#ifndef IB_SA_H #ifndef IB_SA_H
#define IB_SA_H #define IB_SA_H
#include <linux/completion.h>
#include <linux/compiler.h> #include <linux/compiler.h>
#include <asm/atomic.h>
#include <rdma/ib_verbs.h> #include <rdma/ib_verbs.h>
#include <rdma/ib_mad.h> #include <rdma/ib_mad.h>
@ -250,11 +254,28 @@ struct ib_sa_service_rec {
u64 data64[2]; u64 data64[2];
}; };
struct ib_sa_client {
atomic_t users;
struct completion comp;
};
/**
* ib_sa_register_client - Register an SA client.
*/
void ib_sa_register_client(struct ib_sa_client *client);
/**
* ib_sa_unregister_client - Deregister an SA client.
* @client: Client object to deregister.
*/
void ib_sa_unregister_client(struct ib_sa_client *client);
struct ib_sa_query; struct ib_sa_query;
void ib_sa_cancel_query(int id, struct ib_sa_query *query); void ib_sa_cancel_query(int id, struct ib_sa_query *query);
int ib_sa_path_rec_get(struct ib_device *device, u8 port_num, int ib_sa_path_rec_get(struct ib_sa_client *client,
struct ib_device *device, u8 port_num,
struct ib_sa_path_rec *rec, struct ib_sa_path_rec *rec,
ib_sa_comp_mask comp_mask, ib_sa_comp_mask comp_mask,
int timeout_ms, gfp_t gfp_mask, int timeout_ms, gfp_t gfp_mask,
@ -264,7 +285,8 @@ int ib_sa_path_rec_get(struct ib_device *device, u8 port_num,
void *context, void *context,
struct ib_sa_query **query); struct ib_sa_query **query);
int ib_sa_mcmember_rec_query(struct ib_device *device, u8 port_num, int ib_sa_mcmember_rec_query(struct ib_sa_client *client,
struct ib_device *device, u8 port_num,
u8 method, u8 method,
struct ib_sa_mcmember_rec *rec, struct ib_sa_mcmember_rec *rec,
ib_sa_comp_mask comp_mask, ib_sa_comp_mask comp_mask,
@ -275,7 +297,8 @@ int ib_sa_mcmember_rec_query(struct ib_device *device, u8 port_num,
void *context, void *context,
struct ib_sa_query **query); struct ib_sa_query **query);
int ib_sa_service_rec_query(struct ib_device *device, u8 port_num, int ib_sa_service_rec_query(struct ib_sa_client *client,
struct ib_device *device, u8 port_num,
u8 method, u8 method,
struct ib_sa_service_rec *rec, struct ib_sa_service_rec *rec,
ib_sa_comp_mask comp_mask, ib_sa_comp_mask comp_mask,
@ -288,6 +311,7 @@ int ib_sa_service_rec_query(struct ib_device *device, u8 port_num,
/** /**
* ib_sa_mcmember_rec_set - Start an MCMember set query * ib_sa_mcmember_rec_set - Start an MCMember set query
* @client:SA client
* @device:device to send query on * @device:device to send query on
* @port_num: port number to send query on * @port_num: port number to send query on
* @rec:MCMember Record to send in query * @rec:MCMember Record to send in query
@ -311,7 +335,8 @@ int ib_sa_service_rec_query(struct ib_device *device, u8 port_num,
* cancel the query. * cancel the query.
*/ */
static inline int static inline int
ib_sa_mcmember_rec_set(struct ib_device *device, u8 port_num, ib_sa_mcmember_rec_set(struct ib_sa_client *client,
struct ib_device *device, u8 port_num,
struct ib_sa_mcmember_rec *rec, struct ib_sa_mcmember_rec *rec,
ib_sa_comp_mask comp_mask, ib_sa_comp_mask comp_mask,
int timeout_ms, gfp_t gfp_mask, int timeout_ms, gfp_t gfp_mask,
@ -321,7 +346,7 @@ ib_sa_mcmember_rec_set(struct ib_device *device, u8 port_num,
void *context, void *context,
struct ib_sa_query **query) struct ib_sa_query **query)
{ {
return ib_sa_mcmember_rec_query(device, port_num, return ib_sa_mcmember_rec_query(client, device, port_num,
IB_MGMT_METHOD_SET, IB_MGMT_METHOD_SET,
rec, comp_mask, rec, comp_mask,
timeout_ms, gfp_mask, callback, timeout_ms, gfp_mask, callback,
@ -330,6 +355,7 @@ ib_sa_mcmember_rec_set(struct ib_device *device, u8 port_num,
/** /**
* ib_sa_mcmember_rec_delete - Start an MCMember delete query * ib_sa_mcmember_rec_delete - Start an MCMember delete query
* @client:SA client
* @device:device to send query on * @device:device to send query on
* @port_num: port number to send query on * @port_num: port number to send query on
* @rec:MCMember Record to send in query * @rec:MCMember Record to send in query
@ -353,7 +379,8 @@ ib_sa_mcmember_rec_set(struct ib_device *device, u8 port_num,
* cancel the query. * cancel the query.
*/ */
static inline int static inline int
ib_sa_mcmember_rec_delete(struct ib_device *device, u8 port_num, ib_sa_mcmember_rec_delete(struct ib_sa_client *client,
struct ib_device *device, u8 port_num,
struct ib_sa_mcmember_rec *rec, struct ib_sa_mcmember_rec *rec,
ib_sa_comp_mask comp_mask, ib_sa_comp_mask comp_mask,
int timeout_ms, gfp_t gfp_mask, int timeout_ms, gfp_t gfp_mask,
@ -363,7 +390,7 @@ ib_sa_mcmember_rec_delete(struct ib_device *device, u8 port_num,
void *context, void *context,
struct ib_sa_query **query) struct ib_sa_query **query)
{ {
return ib_sa_mcmember_rec_query(device, port_num, return ib_sa_mcmember_rec_query(client, device, port_num,
IB_SA_METHOD_DELETE, IB_SA_METHOD_DELETE,
rec, comp_mask, rec, comp_mask,
timeout_ms, gfp_mask, callback, timeout_ms, gfp_mask, callback,