stack_o2cb.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * stack_o2cb.c
  4. *
  5. * Code which interfaces ocfs2 with the o2cb stack.
  6. *
  7. * Copyright (C) 2007 Oracle. All rights reserved.
  8. */
  9. #include <linux/kernel.h>
  10. #include <linux/crc32.h>
  11. #include <linux/slab.h>
  12. #include <linux/module.h>
  13. /* Needed for AOP_TRUNCATED_PAGE in mlog_errno() */
  14. #include <linux/fs.h>
  15. #include "cluster/masklog.h"
  16. #include "cluster/nodemanager.h"
  17. #include "cluster/heartbeat.h"
  18. #include "cluster/tcp.h"
  19. #include "stackglue.h"
  20. struct o2dlm_private {
  21. struct dlm_eviction_cb op_eviction_cb;
  22. };
  23. static struct ocfs2_stack_plugin o2cb_stack;
  24. /* These should be identical */
  25. #if (DLM_LOCK_IV != LKM_IVMODE)
  26. # error Lock modes do not match
  27. #endif
  28. #if (DLM_LOCK_NL != LKM_NLMODE)
  29. # error Lock modes do not match
  30. #endif
  31. #if (DLM_LOCK_CR != LKM_CRMODE)
  32. # error Lock modes do not match
  33. #endif
  34. #if (DLM_LOCK_CW != LKM_CWMODE)
  35. # error Lock modes do not match
  36. #endif
  37. #if (DLM_LOCK_PR != LKM_PRMODE)
  38. # error Lock modes do not match
  39. #endif
  40. #if (DLM_LOCK_PW != LKM_PWMODE)
  41. # error Lock modes do not match
  42. #endif
  43. #if (DLM_LOCK_EX != LKM_EXMODE)
  44. # error Lock modes do not match
  45. #endif
  46. static inline int mode_to_o2dlm(int mode)
  47. {
  48. BUG_ON(mode > LKM_MAXMODE);
  49. return mode;
  50. }
  51. static int flags_to_o2dlm(u32 flags)
  52. {
  53. int o2dlm_flags = 0;
  54. if (flags & DLM_LKF_NOQUEUE)
  55. o2dlm_flags |= LKM_NOQUEUE;
  56. if (flags & DLM_LKF_CANCEL)
  57. o2dlm_flags |= LKM_CANCEL;
  58. if (flags & DLM_LKF_CONVERT)
  59. o2dlm_flags |= LKM_CONVERT;
  60. if (flags & DLM_LKF_VALBLK)
  61. o2dlm_flags |= LKM_VALBLK;
  62. if (flags & DLM_LKF_IVVALBLK)
  63. o2dlm_flags |= LKM_INVVALBLK;
  64. if (flags & DLM_LKF_ORPHAN)
  65. o2dlm_flags |= LKM_ORPHAN;
  66. if (flags & DLM_LKF_FORCEUNLOCK)
  67. o2dlm_flags |= LKM_FORCE;
  68. if (flags & DLM_LKF_TIMEOUT)
  69. o2dlm_flags |= LKM_TIMEOUT;
  70. if (flags & DLM_LKF_LOCAL)
  71. o2dlm_flags |= LKM_LOCAL;
  72. return o2dlm_flags;
  73. }
  74. /*
  75. * Map an o2dlm status to standard errno values.
  76. *
  77. * o2dlm only uses a handful of these, and returns even fewer to the
  78. * caller. Still, we try to assign sane values to each error.
  79. *
  80. * The following value pairs have special meanings to dlmglue, thus
  81. * the right hand side needs to stay unique - never duplicate the
  82. * mapping elsewhere in the table!
  83. *
  84. * DLM_NORMAL: 0
  85. * DLM_NOTQUEUED: -EAGAIN
  86. * DLM_CANCELGRANT: -EBUSY
  87. * DLM_CANCEL: -DLM_ECANCEL
  88. */
  89. /* Keep in sync with dlmapi.h */
  90. static int status_map[] = {
  91. [DLM_NORMAL] = 0, /* Success */
  92. [DLM_GRANTED] = -EINVAL,
  93. [DLM_DENIED] = -EACCES,
  94. [DLM_DENIED_NOLOCKS] = -EACCES,
  95. [DLM_WORKING] = -EACCES,
  96. [DLM_BLOCKED] = -EINVAL,
  97. [DLM_BLOCKED_ORPHAN] = -EINVAL,
  98. [DLM_DENIED_GRACE_PERIOD] = -EACCES,
  99. [DLM_SYSERR] = -ENOMEM, /* It is what it is */
  100. [DLM_NOSUPPORT] = -EPROTO,
  101. [DLM_CANCELGRANT] = -EBUSY, /* Cancel after grant */
  102. [DLM_IVLOCKID] = -EINVAL,
  103. [DLM_SYNC] = -EINVAL,
  104. [DLM_BADTYPE] = -EINVAL,
  105. [DLM_BADRESOURCE] = -EINVAL,
  106. [DLM_MAXHANDLES] = -ENOMEM,
  107. [DLM_NOCLINFO] = -EINVAL,
  108. [DLM_NOLOCKMGR] = -EINVAL,
  109. [DLM_NOPURGED] = -EINVAL,
  110. [DLM_BADARGS] = -EINVAL,
  111. [DLM_VOID] = -EINVAL,
  112. [DLM_NOTQUEUED] = -EAGAIN, /* Trylock failed */
  113. [DLM_IVBUFLEN] = -EINVAL,
  114. [DLM_CVTUNGRANT] = -EPERM,
  115. [DLM_BADPARAM] = -EINVAL,
  116. [DLM_VALNOTVALID] = -EINVAL,
  117. [DLM_REJECTED] = -EPERM,
  118. [DLM_ABORT] = -EINVAL,
  119. [DLM_CANCEL] = -DLM_ECANCEL, /* Successful cancel */
  120. [DLM_IVRESHANDLE] = -EINVAL,
  121. [DLM_DEADLOCK] = -EDEADLK,
  122. [DLM_DENIED_NOASTS] = -EINVAL,
  123. [DLM_FORWARD] = -EINVAL,
  124. [DLM_TIMEOUT] = -ETIMEDOUT,
  125. [DLM_IVGROUPID] = -EINVAL,
  126. [DLM_VERS_CONFLICT] = -EOPNOTSUPP,
  127. [DLM_BAD_DEVICE_PATH] = -ENOENT,
  128. [DLM_NO_DEVICE_PERMISSION] = -EPERM,
  129. [DLM_NO_CONTROL_DEVICE] = -ENOENT,
  130. [DLM_RECOVERING] = -ENOTCONN,
  131. [DLM_MIGRATING] = -ERESTART,
  132. [DLM_MAXSTATS] = -EINVAL,
  133. };
  134. static int dlm_status_to_errno(enum dlm_status status)
  135. {
  136. BUG_ON(status < 0 || status >= ARRAY_SIZE(status_map));
  137. return status_map[status];
  138. }
  139. static void o2dlm_lock_ast_wrapper(void *astarg)
  140. {
  141. struct ocfs2_dlm_lksb *lksb = astarg;
  142. lksb->lksb_conn->cc_proto->lp_lock_ast(lksb);
  143. }
  144. static void o2dlm_blocking_ast_wrapper(void *astarg, int level)
  145. {
  146. struct ocfs2_dlm_lksb *lksb = astarg;
  147. lksb->lksb_conn->cc_proto->lp_blocking_ast(lksb, level);
  148. }
  149. static void o2dlm_unlock_ast_wrapper(void *astarg, enum dlm_status status)
  150. {
  151. struct ocfs2_dlm_lksb *lksb = astarg;
  152. int error = dlm_status_to_errno(status);
  153. /*
  154. * In o2dlm, you can get both the lock_ast() for the lock being
  155. * granted and the unlock_ast() for the CANCEL failing. A
  156. * successful cancel sends DLM_NORMAL here. If the
  157. * lock grant happened before the cancel arrived, you get
  158. * DLM_CANCELGRANT.
  159. *
  160. * There's no need for the double-ast. If we see DLM_CANCELGRANT,
  161. * we just ignore it. We expect the lock_ast() to handle the
  162. * granted lock.
  163. */
  164. if (status == DLM_CANCELGRANT)
  165. return;
  166. lksb->lksb_conn->cc_proto->lp_unlock_ast(lksb, error);
  167. }
  168. static int o2cb_dlm_lock(struct ocfs2_cluster_connection *conn,
  169. int mode,
  170. struct ocfs2_dlm_lksb *lksb,
  171. u32 flags,
  172. void *name,
  173. unsigned int namelen)
  174. {
  175. enum dlm_status status;
  176. int o2dlm_mode = mode_to_o2dlm(mode);
  177. int o2dlm_flags = flags_to_o2dlm(flags);
  178. int ret;
  179. status = dlmlock(conn->cc_lockspace, o2dlm_mode, &lksb->lksb_o2dlm,
  180. o2dlm_flags, name, namelen,
  181. o2dlm_lock_ast_wrapper, lksb,
  182. o2dlm_blocking_ast_wrapper);
  183. ret = dlm_status_to_errno(status);
  184. return ret;
  185. }
  186. static int o2cb_dlm_unlock(struct ocfs2_cluster_connection *conn,
  187. struct ocfs2_dlm_lksb *lksb,
  188. u32 flags)
  189. {
  190. enum dlm_status status;
  191. int o2dlm_flags = flags_to_o2dlm(flags);
  192. int ret;
  193. status = dlmunlock(conn->cc_lockspace, &lksb->lksb_o2dlm,
  194. o2dlm_flags, o2dlm_unlock_ast_wrapper, lksb);
  195. ret = dlm_status_to_errno(status);
  196. return ret;
  197. }
  198. static int o2cb_dlm_lock_status(struct ocfs2_dlm_lksb *lksb)
  199. {
  200. return dlm_status_to_errno(lksb->lksb_o2dlm.status);
  201. }
  202. /*
  203. * o2dlm aways has a "valid" LVB. If the dlm loses track of the LVB
  204. * contents, it will zero out the LVB. Thus the caller can always trust
  205. * the contents.
  206. */
  207. static int o2cb_dlm_lvb_valid(struct ocfs2_dlm_lksb *lksb)
  208. {
  209. return 1;
  210. }
  211. static void *o2cb_dlm_lvb(struct ocfs2_dlm_lksb *lksb)
  212. {
  213. return (void *)(lksb->lksb_o2dlm.lvb);
  214. }
  215. static void o2cb_dump_lksb(struct ocfs2_dlm_lksb *lksb)
  216. {
  217. dlm_print_one_lock(lksb->lksb_o2dlm.lockid);
  218. }
  219. /*
  220. * Check if this node is heartbeating and is connected to all other
  221. * heartbeating nodes.
  222. */
  223. static int o2cb_cluster_check(void)
  224. {
  225. u8 node_num;
  226. int i;
  227. unsigned long hbmap[BITS_TO_LONGS(O2NM_MAX_NODES)];
  228. unsigned long netmap[BITS_TO_LONGS(O2NM_MAX_NODES)];
  229. node_num = o2nm_this_node();
  230. if (node_num == O2NM_MAX_NODES) {
  231. printk(KERN_ERR "o2cb: This node has not been configured.\n");
  232. return -EINVAL;
  233. }
  234. /*
  235. * o2dlm expects o2net sockets to be created. If not, then
  236. * dlm_join_domain() fails with a stack of errors which are both cryptic
  237. * and incomplete. The idea here is to detect upfront whether we have
  238. * managed to connect to all nodes or not. If not, then list the nodes
  239. * to allow the user to check the configuration (incorrect IP, firewall,
  240. * etc.) Yes, this is racy. But its not the end of the world.
  241. */
  242. #define O2CB_MAP_STABILIZE_COUNT 60
  243. for (i = 0; i < O2CB_MAP_STABILIZE_COUNT; ++i) {
  244. o2hb_fill_node_map(hbmap, sizeof(hbmap));
  245. if (!test_bit(node_num, hbmap)) {
  246. printk(KERN_ERR "o2cb: %s heartbeat has not been "
  247. "started.\n", (o2hb_global_heartbeat_active() ?
  248. "Global" : "Local"));
  249. return -EINVAL;
  250. }
  251. o2net_fill_node_map(netmap, sizeof(netmap));
  252. /* Force set the current node to allow easy compare */
  253. set_bit(node_num, netmap);
  254. if (!memcmp(hbmap, netmap, sizeof(hbmap)))
  255. return 0;
  256. if (i < O2CB_MAP_STABILIZE_COUNT - 1)
  257. msleep(1000);
  258. }
  259. printk(KERN_ERR "o2cb: This node could not connect to nodes:");
  260. i = -1;
  261. while ((i = find_next_bit(hbmap, O2NM_MAX_NODES,
  262. i + 1)) < O2NM_MAX_NODES) {
  263. if (!test_bit(i, netmap))
  264. printk(" %u", i);
  265. }
  266. printk(".\n");
  267. return -ENOTCONN;
  268. }
  269. /*
  270. * Called from the dlm when it's about to evict a node. This is how the
  271. * classic stack signals node death.
  272. */
  273. static void o2dlm_eviction_cb(int node_num, void *data)
  274. {
  275. struct ocfs2_cluster_connection *conn = data;
  276. printk(KERN_NOTICE "o2cb: o2dlm has evicted node %d from domain %.*s\n",
  277. node_num, conn->cc_namelen, conn->cc_name);
  278. conn->cc_recovery_handler(node_num, conn->cc_recovery_data);
  279. }
  280. static int o2cb_cluster_connect(struct ocfs2_cluster_connection *conn)
  281. {
  282. int rc = 0;
  283. u32 dlm_key;
  284. struct dlm_ctxt *dlm;
  285. struct o2dlm_private *priv;
  286. struct dlm_protocol_version fs_version;
  287. BUG_ON(conn == NULL);
  288. BUG_ON(conn->cc_proto == NULL);
  289. /* Ensure cluster stack is up and all nodes are connected */
  290. rc = o2cb_cluster_check();
  291. if (rc) {
  292. printk(KERN_ERR "o2cb: Cluster check failed. Fix errors "
  293. "before retrying.\n");
  294. goto out;
  295. }
  296. priv = kzalloc(sizeof(struct o2dlm_private), GFP_KERNEL);
  297. if (!priv) {
  298. rc = -ENOMEM;
  299. goto out_free;
  300. }
  301. /* This just fills the structure in. It is safe to pass conn. */
  302. dlm_setup_eviction_cb(&priv->op_eviction_cb, o2dlm_eviction_cb,
  303. conn);
  304. conn->cc_private = priv;
  305. /* used by the dlm code to make message headers unique, each
  306. * node in this domain must agree on this. */
  307. dlm_key = crc32_le(0, conn->cc_name, conn->cc_namelen);
  308. fs_version.pv_major = conn->cc_version.pv_major;
  309. fs_version.pv_minor = conn->cc_version.pv_minor;
  310. dlm = dlm_register_domain(conn->cc_name, dlm_key, &fs_version);
  311. if (IS_ERR(dlm)) {
  312. rc = PTR_ERR(dlm);
  313. mlog_errno(rc);
  314. goto out_free;
  315. }
  316. conn->cc_version.pv_major = fs_version.pv_major;
  317. conn->cc_version.pv_minor = fs_version.pv_minor;
  318. conn->cc_lockspace = dlm;
  319. dlm_register_eviction_cb(dlm, &priv->op_eviction_cb);
  320. out_free:
  321. if (rc)
  322. kfree(conn->cc_private);
  323. out:
  324. return rc;
  325. }
  326. static int o2cb_cluster_disconnect(struct ocfs2_cluster_connection *conn)
  327. {
  328. struct dlm_ctxt *dlm = conn->cc_lockspace;
  329. struct o2dlm_private *priv = conn->cc_private;
  330. dlm_unregister_eviction_cb(&priv->op_eviction_cb);
  331. conn->cc_private = NULL;
  332. kfree(priv);
  333. dlm_unregister_domain(dlm);
  334. conn->cc_lockspace = NULL;
  335. return 0;
  336. }
  337. static int o2cb_cluster_this_node(struct ocfs2_cluster_connection *conn,
  338. unsigned int *node)
  339. {
  340. int node_num;
  341. node_num = o2nm_this_node();
  342. if (node_num == O2NM_INVALID_NODE_NUM)
  343. return -ENOENT;
  344. if (node_num >= O2NM_MAX_NODES)
  345. return -EOVERFLOW;
  346. *node = node_num;
  347. return 0;
  348. }
  349. static struct ocfs2_stack_operations o2cb_stack_ops = {
  350. .connect = o2cb_cluster_connect,
  351. .disconnect = o2cb_cluster_disconnect,
  352. .this_node = o2cb_cluster_this_node,
  353. .dlm_lock = o2cb_dlm_lock,
  354. .dlm_unlock = o2cb_dlm_unlock,
  355. .lock_status = o2cb_dlm_lock_status,
  356. .lvb_valid = o2cb_dlm_lvb_valid,
  357. .lock_lvb = o2cb_dlm_lvb,
  358. .dump_lksb = o2cb_dump_lksb,
  359. };
  360. static struct ocfs2_stack_plugin o2cb_stack = {
  361. .sp_name = "o2cb",
  362. .sp_ops = &o2cb_stack_ops,
  363. .sp_owner = THIS_MODULE,
  364. };
  365. static int __init o2cb_stack_init(void)
  366. {
  367. return ocfs2_stack_glue_register(&o2cb_stack);
  368. }
  369. static void __exit o2cb_stack_exit(void)
  370. {
  371. ocfs2_stack_glue_unregister(&o2cb_stack);
  372. }
  373. MODULE_AUTHOR("Oracle");
  374. MODULE_DESCRIPTION("ocfs2 driver for the classic o2cb stack");
  375. MODULE_LICENSE("GPL");
  376. module_init(o2cb_stack_init);
  377. module_exit(o2cb_stack_exit);