/* $Id: ipfail.c,v 1.24.2.7 2005/08/11 21:08:23 gshi Exp $ */
/* ipfail: IP Failover plugin for Linux-HA
*
* Copyright (C) 2002-2004 Kevin Dwyer <kevin@pheared.net>
*
* This plugin uses ping nodes to determine a failure in an
* interface's connectivity and forces a hb_standby. It is based on the
* api_test.c program included with Linux-HA.
*
* Setup: In your ha.cf file make sure you have a ping node setup for each
* interface. Choosing something like the switch that you are connected
* to is a good idea. Choosing your win95 reboot-o-matic is a bad idea.
*
* The way this works is by taking note of when a ping node dies.
* When a death is detected, it communicates with the other side to see
* if the other side saw it die (sort of). If it didn't, then we know
* who deserves to have the resources.
*
* There are ways to improve this, and I'm working on them.
*
*/
/*
* api_test: Test program for testing the heartbeat API
*
* Copyright (C) 2000 Alan Robertson <alanr@unix.sh>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#include <portability.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/utsname.h>
#include <sys/time.h>
#include <sys/stat.h>
#include <stdarg.h>
#include <libgen.h>
#include <heartbeat.h>
#include <ha_msg.h>
#include <hb_api.h>
#include <clplumbing/cl_log.h>
#include <clplumbing/cl_signal.h>
#include <clplumbing/cl_malloc.h>
#include <clplumbing/GSource.h>
#include <clplumbing/Gmain_timeout.h>
#include <clplumbing/coredumps.h>
#include "ipfail.h"
#include <clplumbing/cl_misc.h>
/* ICK! global vars. */
const char *node_name; /* The node we are connected to */
char other_node[SYS_NMLN]; /* The remote node in the pair */
int node_stable; /* Other node stable? */
int need_standby; /* Are we waiting for stability? */
int quitnow = 0; /* Allows a signal to break us out of loop */
int auto_failback; /* How is our auto_failback configured? */
GMainLoop *mainloop; /* Reference to the mainloop for events */
guint delay_giveup_tag = 0;/* Our delay timer */
int
main(int argc, char **argv)
{
unsigned fmask;
ll_cluster_t *hb;
char pid[10];
char *bname, *parameter;
int apifd;
cl_log_enable_stderr(TRUE);
/* Get the name of the binary for logging purposes */
bname = ha_strdup(argv[0]);
cl_log_set_entity(basename(bname));
cl_log_set_facility(DEFAULT_FACILITY);
hb = ll_cluster_new("heartbeat");
memset(other_node, 0, sizeof(other_node));
need_standby = 0;
memset(pid, 0, sizeof(pid));
snprintf(pid, sizeof(pid), "%ld", (long)getpid());
cl_log(LOG_DEBUG, "PID=%s", pid);
open_api(hb);
node_stable = is_stable(hb);
/* Obtain our local node name */
node_name = hb->llc_ops->get_mynodeid(hb);
if (node_name == NULL) {
cl_log(LOG_ERR, "Cannot get my nodeid");
cl_log(LOG_ERR, "REASON: %s", hb->llc_ops->errmsg(hb));
exit(19);
}
cl_log(LOG_DEBUG, "[We are %s]", node_name);
/* Check to see if we should engage auto_failback tactics */
parameter = hb->llc_ops->get_parameter(hb, "auto_failback");
if (parameter) {
/* This is equivalent to nice_failback off */
if (!strcmp(parameter, "legacy")) {
cl_log(LOG_ERR, "auto_failback set to "
"incompatible legacy option.");
exit(100);
}
if(str_to_boolean(parameter, &auto_failback) != HA_OK){
cl_log(LOG_ERR, " invalid auto_faiback value(%s)",
parameter);
exit(100);
}
cl_log(LOG_DEBUG, "auto_failback -> %i (%s)", auto_failback,
parameter);
cl_free(parameter);
}else{
cl_log(LOG_ERR, "Couldn't get auto_failback setting.");
}
/* See if we should drop cores somewhere odd... */
parameter = hb->llc_ops->get_parameter(hb, KEY_COREROOTDIR);
if (parameter) {
cl_set_corerootdir(parameter);
cl_cdtocoredir();
}
cl_cdtocoredir();
set_callbacks(hb);
fmask = LLC_FILTER_DEFAULT;
cl_log(LOG_DEBUG, "Setting message filter mode");
if (hb->llc_ops->setfmode(hb, fmask) != HA_OK) {
cl_log(LOG_ERR, "Cannot set filter mode");
cl_log(LOG_ERR, "REASON: %s", hb->llc_ops->errmsg(hb));
exit(8);
}
node_walk(hb);
set_signals(hb);
cl_log(LOG_DEBUG, "Waiting for messages...");
errno = 0;
cl_log_enable_stderr(FALSE);
/* We will sit in a glib loop waiting for inputs, or making decisions
* for failover
*/
mainloop = g_main_new(TRUE);
apifd = hb->llc_ops->inputfd(hb);
/* Watch the API's fd for input */
G_main_add_fd(G_PRIORITY_HIGH, apifd, FALSE, ipfail_dispatch,
(gpointer)hb, ipfail_dispatch_destroy);
Gmain_timeout_add_full(G_PRIORITY_DEFAULT, 1000,
ipfail_timeout_dispatch, (gpointer)hb,
ipfail_dispatch_destroy);
g_main_run(mainloop);
g_main_destroy(mainloop);
if (!quitnow && errno != EAGAIN && errno != EINTR) {
cl_perror("read_hb_msg returned NULL");
cl_log(LOG_ERR, "REASON: %s", hb->llc_ops->errmsg(hb));
}
close_api(hb);
return 0;
}
int
is_stable(ll_cluster_t *hb)
{
if (!strcmp(hb->llc_ops->get_resources(hb), "transition"))
return 0;
return 1;
}
void
node_walk(ll_cluster_t *hb)
{
const char *node;
/* const char *intf; --Out until ifwalk is fixed */
cl_log(LOG_DEBUG, "Starting node walk");
if (hb->llc_ops->init_nodewalk(hb) != HA_OK) {
cl_log(LOG_ERR, "Cannot start node walk");
cl_log(LOG_ERR, "REASON: %s", hb->llc_ops->errmsg(hb));
exit(9);
}
while((node = hb->llc_ops->nextnode(hb)) != NULL) {
cl_log(LOG_DEBUG, "Cluster node: %s: status: %s", node
, hb->llc_ops->node_status(hb, node));
/* Look for our partner */
if (!strcmp("normal", hb->llc_ops->node_type(hb, node))
&& strcmp(node, node_name)) {
strcpy(other_node, node);
cl_log(LOG_DEBUG, "[They are %s]", other_node);
}
/* ifwalking is broken for ping nodes. I don't think we even
need it at this point.
if (hb->llc_ops->init_ifwalk(hb, node) != HA_OK) {
cl_log(LOG_ERR, "Cannot start if walk");
cl_log(LOG_ERR, "REASON: %s"
, hb->llc_ops->errmsg(hb));
exit(10);
}
while ((intf = hb->llc_ops->nextif(hb))) {
cl_log(LOG_DEBUG, "\tnode %s: intf: %s ifstatus: %s"
, node, intf
, hb->llc_ops->if_status(hb, node, intf));
}
if (hb->llc_ops->end_ifwalk(hb) != HA_OK) {
cl_log(LOG_ERR, "Cannot end if walk");
cl_log(LOG_ERR, "REASON: %s"
, hb->llc_ops->errmsg(hb));
exit(11);
}
-END of ifwalkcode */
}
if (hb->llc_ops->end_nodewalk(hb) != HA_OK) {
cl_log(LOG_ERR, "Cannot end node walk");
cl_log(LOG_ERR, "REASON: %s", hb->llc_ops->errmsg(hb));
exit(12);
}
}
void
set_callbacks(ll_cluster_t *hb)
{
/* Add each of the callbacks we use with the API */
if (hb->llc_ops->set_msg_callback(hb, T_APICLISTAT,
msg_ipfail_join, hb) != HA_OK) {
cl_log(LOG_ERR, "Cannot set msg_ipfail_join callback");
cl_log(LOG_ERR, "REASON: %s", hb->llc_ops->errmsg(hb));
exit(2);
}
if (hb->llc_ops->set_msg_callback(hb, T_RESOURCES,
msg_resources, hb) != HA_OK) {
cl_log(LOG_ERR, "Cannot set msg_resources callback");
cl_log(LOG_ERR, "REASON: %s", hb->llc_ops->errmsg(hb));
exit(18);
}
if (hb->llc_ops->set_msg_callback(hb, "num_ping_nodes",
msg_ping_nodes, hb) != HA_OK) {
cl_log(LOG_ERR, "Cannot set msg callback");
cl_log(LOG_ERR, "REASON: %s", hb->llc_ops->errmsg(hb));
exit(3);
}
if (hb->llc_ops->set_msg_callback(hb, "abort_giveup",
msg_abort_giveup, hb) != HA_OK) {
cl_log(LOG_ERR, "Cannot set msg_abort_giveup callback");
cl_log(LOG_ERR, "REASON: %s", hb->llc_ops->errmsg(hb));
exit(4);
}
if (hb->llc_ops->set_msg_callback(hb, "you_are_dead",
i_am_dead, hb) != HA_OK) {
cl_log(LOG_ERR, "Cannot set i_am_dead callback");
cl_log(LOG_ERR, "REASON: %s", hb->llc_ops->errmsg(hb));
exit(5);
}
if (hb->llc_ops->set_nstatus_callback(hb, NodeStatus, hb) != HA_OK) {
cl_log(LOG_ERR, "Cannot set node status callback");
cl_log(LOG_ERR, "REASON: %s", hb->llc_ops->errmsg(hb));
exit(6);
}
if (hb->llc_ops->set_ifstatus_callback(hb, LinkStatus, hb) != HA_OK) {
cl_log(LOG_ERR, "Cannot set if status callback");
cl_log(LOG_ERR, "REASON: %s", hb->llc_ops->errmsg(hb));
exit(7);
}
}
void
set_signals(ll_cluster_t *hb)
{
/* Setup the various signals */
CL_SIGINTERRUPT(SIGINT, 1);
CL_SIGNAL(SIGINT, gotsig);
CL_SIGINTERRUPT(SIGTERM, 1);
CL_SIGNAL(SIGTERM, gotsig);
cl_log(LOG_DEBUG, "Setting message signal");
if (hb->llc_ops->setmsgsignal(hb, 0) != HA_OK) {
cl_log(LOG_ERR, "Cannot set message signal");
cl_log(LOG_ERR, "REASON: %s", hb->llc_ops->errmsg(hb));
exit(13);
}
}
void
NodeStatus(const char *node, const char *status, void *private)
{
/* Callback for node status changes */
cl_log(LOG_INFO, "Status update: Node %s now has status %s"
, node, status);
if (strcmp(status, DEADSTATUS) == 0) {
if (ping_node_status(private)) {
cl_log(LOG_INFO, "NS: We are still alive!");
} else {
cl_log(LOG_INFO, "NS: We are dead. :<");
}
} else if (strcmp(status, PINGSTATUS) == 0) {
/* A ping node just came up, if we died, request resources?
* If so, that would emulate the primary/secondary type of
* High-Availability, instead of nice_failback mode
*/
/* Lets make sure we weren't both down, and now half up. */
int num_ping;
cl_log(LOG_INFO, "A ping node just came up.");
num_ping = ping_node_status(private);
ask_ping_nodes(private, num_ping);
}
}
void
LinkStatus(const char *node, const char *lnk, const char *status,
void *private)
{
/* Callback for Link status changes */
int num_ping=0;
cl_log(LOG_INFO, "Link Status update: Link %s/%s now has status %s"
, node, lnk, status);
if (strcmp(status, DEADSTATUS) == 0) {
/* If we can still see pinging node, request resources */
if ((num_ping = ping_node_status(private))) {
ask_ping_nodes(private, num_ping);
cl_log(LOG_INFO, "Checking remote count"
" of ping nodes.");
} else {
cl_log(LOG_INFO, "We are dead. :<");
ask_ping_nodes(private, num_ping);
}
}
}
int
ping_node_status(ll_cluster_t *hb)
{
/* ping_node_status: Takes the hearbeat cluster as input,
* returns number of ping nodes found to be in the cluster,
* and therefore alive.
*/
const char *node;
int found=0; /* Number of ping nodes found */
if (hb->llc_ops->init_nodewalk(hb) != HA_OK) {
cl_log(LOG_ERR, "Cannot start node walk");
cl_log(LOG_ERR, "REASON: %s", hb->llc_ops->errmsg(hb));
exit(16);
}
while((node = hb->llc_ops->nextnode(hb))!= NULL) {
if (!strcmp(PINGSTATUS,
hb->llc_ops->node_status(hb, node))) {
cl_log(LOG_DEBUG, "Found ping node %s!", node);
found++;
}
}
if (hb->llc_ops->end_nodewalk(hb) != HA_OK) {
cl_log(LOG_ERR, "Cannot end node walk");
cl_log(LOG_ERR, "REASON: %s", hb->llc_ops->errmsg(hb));
exit(17);
}
return found;
}
gboolean
giveup(gpointer user_data)
{
/* Giveup: Takes the heartbeat cluster as input and the type of
* resources to give up. Returning FALSE causes the timer to die.
* Forces the local node to release a particular class of resources.
*/
struct giveup_data *gd = user_data;
ll_cluster_t *hb = gd->hb;
const char *res_type = gd->res_type;
struct ha_msg *msg;
char pid[10];
cl_log(LOG_INFO, "giveup() called (timeout worked)");
if (is_stable(hb)) {
memset(pid, 0, sizeof(pid));
snprintf(pid, sizeof(pid), "%ld", (long)getpid());
msg = ha_msg_new(3);
ha_msg_add(msg, F_TYPE, T_ASKRESOURCES);
ha_msg_add(msg, F_RESOURCES, res_type);
ha_msg_add(msg, F_ORIG, node_name);
ha_msg_add(msg, F_COMMENT, "me");
hb->llc_ops->sendclustermsg(hb, msg);
cl_log(LOG_DEBUG, "Message [" T_ASKRESOURCES "] sent.");
ha_msg_del(msg);
need_standby = 0;
} else {
need_standby = 1;
}
return FALSE;
}
void
delay_giveup(ll_cluster_t *hb, const char *res_type, int mseconds)
{
struct giveup_data *gd;
gd = malloc(sizeof(struct giveup_data));
if (gd == NULL) {
cl_log(LOG_ERR, "Out of memory, can't giveup.");
return;
}
gd->hb = hb;
gd->res_type = res_type;
/* Set mseconds to -1 to use default. (twice the keepalive) */
if (mseconds < 0) {
mseconds = hb->llc_ops->get_keepalive(hb) * 2;
}
cl_log(LOG_INFO, "Delayed giveup in %i seconds.", mseconds / 1000);
if (delay_giveup_tag) {
/* A timer exists already? */
cl_log(LOG_DEBUG, "Detected existing delay timer, overriding");
Gmain_timeout_remove(delay_giveup_tag);
delay_giveup_tag = 0;
}
/* We are going to call giveup in mseconds/1000 Seconds. */
delay_giveup_tag = Gmain_timeout_add_full(G_PRIORITY_DEFAULT,
mseconds,
giveup, (gpointer)gd,
giveup_destroy);
}
void
giveup_destroy(gpointer user_data)
{
/* Clean up the struct giveup_data that we were using */
free(user_data);
delay_giveup_tag = 0;
cl_log(LOG_DEBUG, "giveup timeout has been destroyed.");
}
void
abort_giveup()
{
if (delay_giveup_tag) {
cl_log(LOG_INFO, "Aborted delayed giveup (%u)",
delay_giveup_tag);
Gmain_timeout_remove(delay_giveup_tag);
delay_giveup_tag = 0;
} else {
cl_log(LOG_INFO, "No giveup timer to abort.");
}
}
void
send_abort_giveup(ll_cluster_t *hb)
{
struct ha_msg *msg;
msg = ha_msg_new(2);
ha_msg_add(msg, F_TYPE, "abort_giveup");
ha_msg_add(msg, F_ORIG, node_name);
hb->llc_ops->sendnodemsg(hb, msg, other_node);
cl_log(LOG_DEBUG, "Abort message sent.");
ha_msg_del(msg);
}
void
msg_abort_giveup(const struct ha_msg *msg, void *private)
{
abort_giveup();
}
void
msg_ipfail_join(const struct ha_msg *msg, void *private)
{
/* msg_ipfail_join: When another ipfail client sends a join
* message, call ask_ping_nodes() to compare ping node counts.
* Callback for the T_APICLISTAT message.
*/
/* If this is a join message from ipfail on a different node.... */
if (!strcmp(ha_msg_value(msg, F_STATUS), JOINSTATUS) &&
!strcmp(ha_msg_value(msg, F_FROMID), "ipfail") &&
strcmp(ha_msg_value(msg, F_ORIG), node_name)) {
cl_log(LOG_DEBUG,
"Got join message from another ipfail client. (%s)",
ha_msg_value(msg, F_ORIG));
ask_ping_nodes(private, ping_node_status(private));
}
}
void
msg_resources(const struct ha_msg *msg, void *private)
{
const char * orig = ha_msg_value(msg, F_ORIG);
const char * isstable = ha_msg_value(msg, F_ISSTABLE);
/* msg_resources: Catch T_RESOURCES messages, so that we can
* find out when stability is achieved among the cluster
*/
if (!orig || !isstable) {
return; /* No stability info in this message... */
}
/* Right now there are two stable messages sent out, we are
* only concerned with the one that has no info= line on it.
*/
if (!strcmp(orig, other_node) &&
!ha_msg_value(msg, F_COMMENT) &&
!strcmp(isstable, "1")) {
cl_log(LOG_DEBUG, "Other side is now stable.");
node_stable = 1;
/* There may be a pending standby */
if (need_standby) {
/* Gratuitious ARPs take some time, is there a
* way to know when they're finished? I don't
* want this sleep here, even if it only is during
* startup.
*/
/* This value is prone to be wrong for different
* situations. We need the resource stability
* message to be delayed until the resource scripts
* finish, and then we can stop waiting.
*/
sleep(10);
/* If the resource message stuff is solved, we could
* safely giveup() here. However, since we're waiting
* for arbitrary amounts of time it may be wise to
* recheck the assumptions of the cluster and count
* ping nodes.
*/
ask_ping_nodes(private, ping_node_status(private));
/*giveup(private); */
/* The ask_ping_nodes message will sort out whether
* a standby is necessary. */
need_standby = 0;
}
}
else if (!strcmp(orig, other_node) &&
!strcmp(isstable, "0")) {
cl_log(LOG_DEBUG, "Other side is unstable.");
node_stable = 0;
}
}
void
ask_ping_nodes(ll_cluster_t *hb, int num_ping)
{
/* ask_ping_nodes: Takes the heartbeat cluster and the number of
* ping nodes we can see alive as input, returning nothing.
* It asks the other node for the number of ping nodes it can see.
*/
struct ha_msg *msg;
char pid[10], np[5];
cl_log(LOG_INFO, "Asking other side for ping node count.");
memset(pid, 0, sizeof(pid));
snprintf(pid, sizeof(pid), "%ld", (long)getpid());
memset(np, 0, sizeof(np));
snprintf(np, sizeof(np), "%d", num_ping);
msg = ha_msg_new(3);
ha_msg_add(msg, F_TYPE, "num_ping_nodes");
ha_msg_add(msg, F_ORIG, node_name);
ha_msg_add(msg, F_NUMPING, np);
hb->llc_ops->sendnodemsg(hb, msg, other_node);
cl_log(LOG_DEBUG, "Message [" F_NUMPING "] sent.");
ha_msg_del(msg);
}
void
msg_ping_nodes(const struct ha_msg *msg, void *private)
{
/* msg_ping_nodes: Takes the message and heartbeat cluster as input;
* returns nothing. Callback for the num_ping_nodes message.
*/
int num_nodes=0;
ll_cluster_t *hb = private;
cl_log(LOG_DEBUG, "Got asked for num_ping.");
num_nodes = ping_node_status(hb);
if (num_nodes > atoi(ha_msg_value(msg, F_NUMPING))) {
cl_log(LOG_INFO,
"Telling other node that we have more visible ping "
"nodes.");
you_are_dead(hb);
}
else if (num_nodes < atoi(ha_msg_value(msg, F_NUMPING))) {
cl_log(LOG_INFO,
"Giving up because we have less visible ping nodes.");
delay_giveup(hb, HB_ALL_RESOURCES, -1);
}
else {
cl_log(LOG_INFO, "Ping node count is balanced.");
send_abort_giveup(hb);
if (delay_giveup_tag) {
/* We've got a delayed giveup, and we're now balanced*/
/* BUG? We don't want to do this if we have an
auto_failback pending, I think. */
abort_giveup();
} else if (auto_failback && is_stable(hb)) {
/* We're balanced, so make sure we don't have foreign
* stuff
*/
cl_log(LOG_INFO,
"Giving up foreign resources (auto_failback).");
delay_giveup(hb, HB_FOREIGN_RESOURCES, -1);
}
}
}
void
you_are_dead(ll_cluster_t *hb)
{
/* you_are_dead: Takes the heartbeat cluster as input; returns nothing.
* Sends the you_are_dead message to the dead node.
*/
struct ha_msg *msg;
char pid[10];
cl_log(LOG_DEBUG, "Sending you_are_dead.");
memset(pid, 0, sizeof(pid));
snprintf(pid, sizeof(pid), "%ld", (long)getpid());
msg = ha_msg_new(1);
ha_msg_add(msg, F_TYPE, "you_are_dead");
hb->llc_ops->sendnodemsg(hb, msg, other_node);
cl_log(LOG_DEBUG, "Message [you_are_dead] sent.");
ha_msg_del(msg);
}
void
i_am_dead(const struct ha_msg *msg, void *private)
{
/* i_am_dead: Takes the you_are_dead message and the heartbeat cluster
* as input; returns nothing.
* Callback for the you_are_dead message.
*/
cl_log(LOG_INFO,
"Giving up because we were told that we have less ping nodes.");
delay_giveup(private, HB_ALL_RESOURCES, -1);
}
void
gotsig(int nsig)
{
(void)nsig;
quitnow = 1;
}
/* Used to handle the API in the gmainloop */
gboolean
ipfail_dispatch(int fd, gpointer user_data)
{
struct ha_msg *reply;
ll_cluster_t *hb = user_data;
/*
if (hb->llc_ops->msgready(hb))
cl_log(LOG_DEBUG, "Msg ready!");
cl_log(LOG_DEBUG, "Reading a message!");
*/
reply = hb->llc_ops->readmsg(hb, 0);
if (reply != NULL) {
/* cl_log_message(reply); */
ha_msg_del(reply); reply=NULL;
return TRUE;
}
/*
else
return FALSE;
*/
return TRUE;
}
void
ipfail_dispatch_destroy(gpointer user_data)
{
return;
}
gboolean
ipfail_timeout_dispatch(gpointer user_data)
{
ll_cluster_t *hb = user_data;
if (quitnow) {
g_main_quit(mainloop);
return FALSE;
}
if (hb->llc_ops->msgready(hb)) {
/* cl_log(LOG_DEBUG, "Msg ready! [2]"); */
return ipfail_dispatch(-1, user_data);
}
return TRUE;
}
void
open_api(ll_cluster_t *hb)
{
/* Sign in to the API and setup the log facility */
int facility;
cl_log(LOG_DEBUG, "Signing in with heartbeat");
if (hb->llc_ops->signon(hb, "ipfail")!= HA_OK) {
cl_log(LOG_ERR, "Cannot sign on with heartbeat");
cl_log(LOG_ERR, "REASON: %s", hb->llc_ops->errmsg(hb));
exit(1);
}
if ((facility = hb->llc_ops->get_logfacility(hb)) <= 0) {
facility = DEFAULT_FACILITY;
}
cl_log_set_facility(facility);
}
void
close_api(ll_cluster_t *hb)
{
/* Log off of the API and clean up */
if (hb->llc_ops->signoff(hb) != HA_OK) {
cl_log(LOG_ERR, "Cannot sign off from heartbeat.");
cl_log(LOG_ERR, "REASON: %s", hb->llc_ops->errmsg(hb));
exit(14);
}
if (hb->llc_ops->delete(hb) != HA_OK) {
cl_log(LOG_ERR, "REASON: %s", hb->llc_ops->errmsg(hb));
cl_log(LOG_ERR, "Cannot delete API object.");
exit(15);
}
}
syntax highlighted by Code2HTML, v. 0.9.1